Code:
package com.asteria.utility;
import java.io.FileWriter;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.logging.Logger;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import com.asteria.game.character.npc.NpcDefinition;
import com.asteria.game.character.npc.drop.NpcDrop;
import com.asteria.game.character.npc.drop.NpcDropChance;
import com.asteria.game.character.npc.drop.NpcDropTable;
import com.asteria.game.item.ItemDefinition;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
/**
* Npc drop grabber - grabs drops from RS WIKI
*
* @author Cadillac
*/
public class NpcDropGrabber {
/* Whether or not to print messages for debugging. */
private static boolean debug = true;
/* The maximum item id */
private static final int MAX_ITEM_ID = 20694;
/* The npc names to grab drops for */
private static final List<String> NPC_NAMES = new LinkedList<>();
/* The logger */
Logger log = LoggerUtils.getLogger(NpcDropGrabber.class);
public void determineNpcs() {
for (NpcDefinition def : NpcDefinition.DEFINITIONS) {
if (def == null || def.getName() == null) {
continue;
}
if (def.isAttackable() && def.getHitpoints() > 0) {
NPC_NAMES.add(def.getName());
}
}
}
public void run() {
determineNpcs();
// hashmap because I was having touble parsing without specifying
HashMap<String, NpcDropTable> drops = new HashMap<>();
for (String name : NPC_NAMES) {
NpcDropTable table = grab(name);
if (table == null) {
continue;
}
drops.put(name, table);
}
dump(drops);
}
/**
* Grabs the {@link NpcDropTable} corresponding to the npc name. <br/>
* *** <b> BE SURE TO USE NAMES EXACTLY AS THEY APPEAR IN DEFINITIONS </b>
*
* @param name
*/
public NpcDropTable grab(String name) {
String key = name;
// Format the name to append to the url.
name = TextUtils.capitalize(name.toLowerCase().replaceAll(" ", "_"));
List<NpcDrop> dumped = new ArrayList<>();
try {
Document page = Jsoup.connect("http://runescape.wikia.com/wiki/" + name).get();
System.out.println(page.title());
// Get the item drop elements (tables) from 'page'
Elements dropRows = page.select(".item-drops");
//log.info(dropRows.size());
if (dropRows.size() < 1) {
log.info("No drop table for " + name);
return null;
}
// Begin looping all of the tables.
for (Element row : dropRows) {
//debug(row);
// The table itself.
Element table = row.children().get(0);
//log.info("Amount of children: " + table.children().size());
// We are at the table rows now and can begin dissecting drops.
for (Element tableRow : table.select("tr")) {
//debug(tableRow);
// Size of TD (Table Data) which are the row components http://i.imgur.com/RLc093a.png
Elements tableData = tableRow.select("td");
//log.info("Size of TD: " + tableData.size());
// Html which we use for the rarity and amount(s)
String html = tableData.html();
//log.info("" + html);
// The a(nchor) element has first, the image, second, the item name
Elements links = tableData.select("a");
//log.info("Links: " + links.size());
String itemName = "";
// for some reason links.get(1) throws an error...
int accumulator = 0;
for (Element link : links) {
if (accumulator == 1) {
itemName = link.text();
}
accumulator++;
}
// Skip if null
if (itemName.equals(null)) {
continue;
}
Optional<ItemDefinition> def = ItemDefinition.get(itemName);
// Skip if we don't have a definiton for this item.
if (!def.isPresent() || def.get().getId() > MAX_ITEM_ID) {
log.info("skipped " + itemName);
continue;
}
String[] lines = html.split("\n");
// The amount of lines where we extract the amoutn and rarity from
//log.info("Html Lines " + lines.length);
String amount = "";
String rarity = "";
int index = 0;
// Looping because yet again calling lines[5] lines[6] is throwing errors..
for (String line : lines) {
if (index == 5) { // The line id where we get the amount(s)
amount = line.replaceAll(",", "");
} else if (index == 6) { // The line with rarity
rarity = line;
}
index++;
}
// The determined drop chance
NpcDropChance chance = NpcDropChance.get(rarity);
// The item id and note id.
int id = def.get().getId();
int child = def.get().getNoteId();
// Now we actually gather the possible drops.
if (amount.contains(";")) {
String[] split = amount.split(";");
for (String s : split) {
String trimmed = s.replaceAll(" ", "");
// The drop has a varying amount
if (s.contains("–")) {
String[] values = s.split("–");
boolean note = values[0].contains("note") || values[1].contains("note");
int min = Integer.parseInt(values[0].replaceAll("[^\\d.]", ""));
int max = Integer.parseInt(values[1].replaceAll("[^\\d.]", ""));
dumped.add(new NpcDrop(note ? child : id, min, max, chance));
// Handle the noted amounts
} else if (amount.contains("noted")) {
String amt = amount.replaceAll("[^\\d.]", "");
int a = Integer.parseInt(amt.replaceAll(" ", ""));
dumped.add(new NpcDrop(child, a, a, chance));
// An unkown amount so we just set it to 1
} else if (amount.equalsIgnoreCase("Unknown")){
dumped.add(new NpcDrop(id, 1, 1, chance));
// The drop has multiple set amounts
} else {
String amt = s.replaceAll("[^\\d.]", "");
if (amt.equals("")) {
continue;
}
int a = Integer.parseInt(amt);
dumped.add(new NpcDrop(id, a, a, chance));
}
}
// The dropped item is noted so we drop the noted id
} else if (amount.contains("noted")) {
String amt = amount.replaceAll("[^\\d.]", "");
int a = Integer.parseInt(amt.replaceAll(" ", ""));
dumped.add(new NpcDrop(child, a, a, chance));
// A single drop with inclusive values
} else if (amount.contains("–")) {
String[] values = amount.split("–");
boolean note = values[0].contains("note") || values[1].contains("note");
int min = Integer.parseInt(values[0].replaceAll("[^\\d.]", ""));
int max = Integer.parseInt(values[1].replaceAll("[^\\d.]", ""));
dumped.add(new NpcDrop(note ? child : id, min, max, chance));
// A static drop
} else if (amount.equalsIgnoreCase("Unknown")){
dumped.add(new NpcDrop(id, 1, 1, chance));
} else {
if (amount.equals("")) {
continue;
}
int a = Integer.parseInt(amount.replaceAll("[^\\d.]", ""));
dumped.add(new NpcDrop(id, a, a, chance));
}
log.info("Item Name: " + itemName + " Amount: " + amount + " Rarity : " + rarity);
// NpcDrop drop = ItemDefinitions.get(itemName);
//List.add(drop);
}
}
} catch (IOException e) {
//System.exit(1);
e.printStackTrace();
}
log.info("Drops dumped: " + dumped.size());
// no drops so this is null.
if (dumped.size() < 1) {
return null;
}
List<NpcDrop> constants = new ArrayList<>();
List<NpcDrop> drops = new ArrayList<>();
for (NpcDrop drop : dumped) {
if (drop.getChance() == NpcDropChance.ALWAYS) {
constants.add(drop);
} else {
drops.add(drop);
}
}
NpcDrop[] constantArray = constants.toArray(new NpcDrop[constants.size()]);
NpcDrop[] dropArray = drops.toArray(new NpcDrop[drops.size()]);
return new NpcDropTable(key, constantArray, dropArray);
}
public void dump(HashMap<String, NpcDropTable> map) {
// Create a json pretty print object.
Gson gson = new GsonBuilder().setPrettyPrinting().create();
//
String json = gson.toJson(map);
try {
FileWriter writer = new FileWriter("./npc-drops.json");
writer.write(json);
writer.close();
} catch (IOException e) {
e.printStackTrace();
}
}
private void debug(Element element) {
if (debug)
log.info(element.nodeName() + " Child Nodes: " + element.childNodeSize() + " Class Name: " + element.className() + " Node Name: " + element.nodeName());
}
}