Thread: Osrs wiki drop table dumper

Results 1 to 8 of 8
  1. #1 Osrs wiki drop table dumper 
    Owner of Dawntained

    Mgt Madness's Avatar
    Join Date
    Oct 2011
    Age
    28
    Posts
    3,380
    Thanks given
    1,429
    Thanks received
    958
    Rep Power
    2168
    Code:
    package utility;
    
    import game.item.ItemAssistant;
    import game.item.ItemDefinition;
    import java.io.IOException;
    import java.util.ArrayList;
    import org.jsoup.Jsoup;
    import org.jsoup.nodes.Document;
    import org.jsoup.nodes.Element;
    import org.jsoup.select.Elements;
    
    public class RsWikiDropDumper {
    
    	/**
    	 * Name of the npc, the program will auto format your input, if it formatted it incorrectly, fill up UNFORMATTED_NAME below instead.
    	 * <p>If you type in kree'arra on this, it will format it to Kree'Arra which is not a valid url, use the UNFORMATTED_NAME below and type in Kree'arra.
    	 */
    	public final static String NAME_TO_FORMAT = "abyssal demon";
    
    	/**
    	 * If this is not an empty string, it will force use this name to search up the npc without formatting the string.
    	 */
    	public final static String UNFORMATTED_NAME = "";
    
    	public final static boolean DEBUG = false;
    
    	public static void main(String args[]) {
    		ItemDefinition.loadItemDefinitionsAll();
    
    		// Format the name to append to the url.
    		String name = Misc.capitalize(NAME_TO_FORMAT.toLowerCase().replaceAll(" ", "_"));
    		if (!UNFORMATTED_NAME.isEmpty()) {
    			name = UNFORMATTED_NAME;
    		}
    		try {
    
    			ArrayList<String> errors = new ArrayList<>();
    			ArrayList<String> drops = new ArrayList<>();
    			Document page = Jsoup.connect("http://oldschoolrunescape.wikia.com/wiki/" + name).get();
    
    			Misc.print("Dumped from: " + "http://oldschoolrunescape.wikia.com/wiki/" + name);
    
    			// Get the item drop elements (tables) from 'page'
    			Elements dropRows = page.select(".dropstable");
    
    			if (dropRows.size() < 1) {
    				Misc.print("No drop table for " + name);
    				return;
    			}
    
    			// Begin looping all of the tables.
    			for (Element row : dropRows) {
    
    				// The table itself.
    				Element table = row.children().get(0);
    
    				// We are at the table rows now and can begin dissecting drops.
    				for (Element tableRow : table.select("tr")) {
    					if (tableRow.toString().contains("Show/hide rare drop table")) {
    						break;
    					}
    
    					// Size of TD (Table Data) which are the row components http://i.imgur.com/RLc093a.png
    					Elements tableData = tableRow.select("td");
    
    					// Html which we use for the rarity and amount(s)
    					String html = tableData.html();
    
    					// The a(nchor) element has first, the image, second, the item name
    					Elements links = tableData.select("a");
    
    					String itemName = "";
    
    					// for some reason links.get(1) throws an error...
    					int accumulator = 0;
    					for (Element link : links) {
    						if (accumulator == 1) {
    							itemName = link.text();
    						}
    						accumulator++;
    					}
    
    					// Skip if null
    					if (itemName.isEmpty()) {
    						continue;
    					}
    
    					int itemIdFound = ItemAssistant.getItemId(itemName, false);
    					// Skip if we don't have a definiton for this item.
    					if (itemIdFound < 0) {
    						errors.add("Skipped item name: " + itemName);
    						continue;
    					}
    
    					String[] lines = html.split("\n");
    
    					// The amount of lines where we extract the amoutn and rarity from
    
    					String amount = "";
    					String rarity = "";
    
    					int index = 0;
    
    					// Looping because yet again calling lines[5] lines[6] is throwing errors..
    					for (String line : lines) {
    
    						if (index == 5) { // The line id where we get the amount(s)
    							amount = line.replaceAll(",", "");
    						} else if (index == 6) { // The line with rarity
    							rarity = line;
    						}
    
    						index++;
    					}
    
    					// The determined drop chance
    					int chance = getChanceDependingOnRarity(rarity);
    					if (chance <= 0) {
    						errors.add("Incorrect chance: " + chance + " for: " + itemName + ", " + ItemAssistant.getItemName(itemIdFound));
    						continue;
    					}
    
    					// The item id and note id.
    					int normalItemId = itemIdFound;
    					int notedItemId = ItemAssistant.getNotedItem(itemIdFound);
    
    					// Now we actually gather the possible drops.
    					if (amount.contains(";")) {
    						String[] split = amount.split(";");
    						for (String s : split) {
    							// The drop has a varying amount
    							if (s.contains("�")) {
    								String[] values = s.split("�");
    								boolean note = values[0].contains("note") || values[1].contains("note");
    								int min = Integer.parseInt(values[0].replaceAll("[^\\d.]", ""));
    								int max = Integer.parseInt(values[1].replaceAll("[^\\d.]", ""));
    								int finalItemId = note ? notedItemId : normalItemId;
    								if (DEBUG) {
    									Misc.print("Here7: " + finalItemId);
    								}
    								drops.add(chance + " " + finalItemId + " " + min + "-" + max + " // " + ItemAssistant.getItemName(finalItemId));
    								// Handle the noted amounts
    							} else if (amount.contains("noted")) {
    								String amt = amount.replaceAll("[^\\d.]", "");
    								int a = Integer.parseInt(amt.replaceAll(" ", ""));
    								if (DEBUG) {
    									Misc.print("Here6: " + notedItemId);
    								}
    								drops.add(chance + " " + notedItemId + " " + a + " // " + ItemAssistant.getItemName(notedItemId));
    								// An unkown amount so we just set it to 1
    							} else if (amount.equalsIgnoreCase("Unknown")) {
    								if (DEBUG) {
    									Misc.print("Here5: " + normalItemId);
    								}
    								drops.add(chance + " " + normalItemId + " " + 1 + " // " + ItemAssistant.getItemName(normalItemId));
    								// The drop has multiple set amounts
    							} else {
    								String amt = s.replaceAll("[^\\d.]", "");
    								if (amt.equals("")) {
    									continue;
    								}
    								int a = Integer.parseInt(amt);
    								if (DEBUG) {
    									Misc.print("Here4: " + normalItemId);
    								}
    								drops.add(chance + " " + normalItemId + " " + a + " // " + ItemAssistant.getItemName(normalItemId));
    							}
    						}
    						// The dropped item is noted so we drop the noted id
    					} else if (amount.contains("noted")) {
    						String amt = amount.replaceAll("[^\\d.]", " ");
    						amt = amt.replaceFirst(" ", "-");
    						amt = amt.trim();
    						if (amt.endsWith("-")) {
    							amt = amt.substring(0, amt.length() - 1);
    						}
    						if (DEBUG) {
    							Misc.print("Here3: " + notedItemId);
    						}
    						drops.add(chance + " " + notedItemId + " " + amt + " // " + ItemAssistant.getItemName(notedItemId));
    						// A single drop with inclusive values
    					} else if (amount.contains("�")) {
    						String[] values = amount.split("�");
    						boolean note = values[0].contains("note") || values[1].contains("note");
    						int min = Integer.parseInt(values[0].replaceAll("[^\\d.]", ""));
    						int max = Integer.parseInt(values[1].replaceAll("[^\\d.]", ""));
    						int finalItemId = note ? notedItemId : normalItemId;
    						if (DEBUG) {
    							Misc.print("Here2: " + finalItemId);
    						}
    						drops.add(chance + " " + finalItemId + " " + min + "-" + max + " // " + ItemAssistant.getItemName(finalItemId));
    						// A static drop
    					} else if (amount.equalsIgnoreCase("Unknown")) {
    						drops.add(chance + " " + normalItemId + " " + 1 + " // " + ItemAssistant.getItemName(normalItemId));
    					} else {
    						if (amount.isEmpty()) {
    							errors.add("Empty amount: " + itemName);
    							continue;
    						}
    						String itemAmount = amount.replaceAll("[^\\d.]", "");
    						if (amount.contains("–")) { // this dash is not the same as the normal dash -
    							itemAmount = amount.replace("–", "-")
    						}
    						if (DEBUG) {
    							Misc.print("Here1: " + normalItemId + ", " + itemAmount + ", " + amount);
    						}
    						drops.add(chance + " " + normalItemId + " " + itemAmount + " // " + ItemAssistant.getItemName(normalItemId));
    					}
    				}
    			}
    			if (drops.isEmpty()) {
    				Misc.print("No drops found for: " + name);
    			} else {
    				for (int counter = 0; counter < drops.size() - 1; counter++) {
    					for (int index = 0; index < drops.size() - 1 - counter; index++) {
    						String line1 = drops.get(index);
    						String[] parse1 = line1.split(" ");
    						int rarity1 = Integer.parseInt(parse1[0]);
    						String line2 = drops.get(index + 1);
    						String[] parse2 = line2.split(" ");
    						int rarity2 = Integer.parseInt(parse2[0]);
    						if (rarity1 > rarity2) {
    							String temp = line1;
    							drops.remove(index);
    							drops.add(index, line2);
    							drops.remove(index + 1);
    							drops.add(index + 1, temp);
    						}
    					}
    				}
    				for (int index = drops.size() - 1; index > -1; index--) {
    					Misc.print(drops.get(index));
    				}
    			}
    			if (!errors.isEmpty()) {
    				Misc.print("Errors----");
    			}
    			for (int index = 0; index < errors.size(); index++) {
    				Misc.print(errors.get(index));
    			}
    
    		} catch (IOException e) {
    			e.printStackTrace();
    		}
    
    	}
    
    	public final static int ALWAYS = 1;
    
    	public final static int COMMON = 8;
    
    	public final static int UNCOMMON = 32;
    
    	public final static int RARE = 128;
    
    	public final static int VERY_RARE = 256;
    
    	private static int getChanceDependingOnRarity(String rarity) {
    		switch (rarity) {
    			case "Always":
    				return ALWAYS;
    			case "Common":
    				return COMMON;
    			case "Uncommon":
    				return UNCOMMON;
    			case "Rare":
    				return RARE;
    			case "Very rare":
    				return VERY_RARE;
    		}
    		if (rarity.contains("/")) {
    			String shortVersion = rarity.substring(0, 23);
    			if (!shortVersion.contains("<small>")) {
    				if (rarity.contains("Always")) {
    					return ALWAYS;
    				}
    				if (rarity.contains("Common")) {
    					return COMMON;
    				}
    				if (rarity.contains("Uncommon")) {
    					return UNCOMMON;
    				}
    				if (rarity.contains("Rare")) {
    					return RARE;
    				}
    				if (rarity.contains("Very rare")) {
    					return VERY_RARE;
    				}
    			}
    			//Uncommon (3/128)
    			rarity = rarity.replace("(", "");
    			rarity = rarity.replace(")", "");
    			rarity = rarity.substring(rarity.indexOf(" "));
    			int start = rarity.indexOf(">") + 1;
    			int end = 0;
    			int chopped = rarity.indexOf("/");
    			String temp = rarity.substring(chopped);
    			end = temp.indexOf("<") + chopped;
    			rarity = rarity.substring(start, end);
    			rarity = rarity.replaceAll("~", "");
    			rarity = rarity.replaceAll(",", "");
    			String[] parse = rarity.split("/");
    			double firstNumber = Double.parseDouble(parse[0]);
    			double secondNumber = Double.parseDouble(parse[1]);
    			return (int) (secondNumber / firstNumber);
    		}
    		return -1;
    	}
    }
    Output:
    Code:
    Dumped from: http://oldschoolrunescape.wikia.com/wiki/Abyssal_Demon
    32768 13265 1 // Abyssal dagger
    6000 7979 1 // Abyssal head
    1200 12073 1 // Clue scroll (elite)
    512 4151 1 // Abyssal whip
    350 19683 1 // Dark totem top
    350 19681 1 // Dark totem middle
    350 19679 1 // Dark totem base
    233 19677 1 // Ancient shard
    128 12542 1 // Clue scroll (hard)
    128 217 1 // Grimy dwarf weed
    128 2485 1 // Grimy lantadyme
    128 215 1 // Grimy cadantine
    32 133 1 // Defence potion(3)
    32 2361 1 // Adamantite bar
    32 1452 1 // Chaos talisman
    32 1454 1 // Cosmic talisman
    32 563 3 // Law rune
    32 213 1 // Grimy kwuarm
    32 211 1 // Grimy avantoe
    32 209 1 // Grimy irit leaf
    32 207 1 // Grimy ranarr weed
    32 205 1 // Grimy harralander
    32 1147 1 // Rune med helm
    32 1113 1 // Rune chainbody
    32 1197 1 // Mithril kiteshield
    25 13507 1 // Ensouled abyssal head
    8 7937 60 // Pure essence
    8 379 1 // Lobster
    8 995 460 // Coins
    8 995 220 // Coins
    8 995 132 // Coins
    8 995 44 // Coins
    8 995 30 // Coins
    8 562 10 // Chaos rune
    8 565 7 // Blood rune
    8 556 50 // Air rune
    8 203 1 // Grimy tarromin
    8 201 1 // Grimy marrentill
    8 199 1 // Grimy guam leaf
    8 1365 1 // Steel battleaxe
    8 1361 1 // Black axe
    8 1283 1 // Black sword
    1 592 1 // Ashes
    I won't be offering any help because of my busy schedule.
    Enjoy.
    The code is based off this thread who released it for RS Wiki drop table dumper instead.
    Last edited by Mgt Madness; 07-22-2018 at 09:15 AM.
    Attached image
    Reply With Quote  
     

  2. Thankful users:


  3. #2  
    ¯\_(ツ)_/¯


    Join Date
    Jul 2014
    Posts
    1,803
    Thanks given
    928
    Thanks received
    550
    Rep Power
    299
    Thanks for this
    Reply With Quote  
     

  4. #3  
    Respected Member


    Join Date
    Jan 2009
    Posts
    5,743
    Thanks given
    1,162
    Thanks received
    3,603
    Rep Power
    5000
    another awful wiki dumper which doesnt use the xml dump - seconds rather than minutes/hours....
    Reply With Quote  
     

  5. Thankful users:


  6. #4  
    Donator

    TeJay's Avatar
    Join Date
    Jul 2017
    Posts
    630
    Thanks given
    217
    Thanks received
    283
    Rep Power
    1754
    Not bad. Keep the good work up, try working on xml dump.
    Attached image
    Spoiler for Services:
    Attached image
    Reply With Quote  
     

  7. #5  
    Renown Programmer & Respected Member

    Ryley's Avatar
    Join Date
    Aug 2011
    Posts
    596
    Thanks given
    254
    Thanks received
    521
    Rep Power
    1332
    pls dont parse html why are you making your life difficult
    Reply With Quote  
     

  8. Thankful user:


  9. #6  
    Respected Member


    Join Date
    Jul 2015
    Posts
    781
    Thanks given
    206
    Thanks received
    394
    Rep Power
    524
    I know in the past the XML dump wasn't updated frequently.. so I actually prefer parsing the tables. GJ..
    Reply With Quote  
     

  10. #7  
    Blurite

    Corey's Avatar
    Join Date
    Feb 2012
    Age
    26
    Posts
    1,491
    Thanks given
    1,245
    Thanks received
    1,729
    Rep Power
    5000
    Code can deffo be improved, and use XML instead.
    Although to be fair the XML dump hasn't been updated in 3 months, so if you download it you'll get the Wikia from April.

    I'll write a Kotlin, XML version of this and release it at some point.
    Attached image
    Reply With Quote  
     

  11. #8  
    Registered Member
    Join Date
    Jan 2018
    Posts
    136
    Thanks given
    143
    Thanks received
    10
    Rep Power
    58
    Thank you!
    Possibly to do one for definitions in json format for Elvarg at all please?
    Reply With Quote  
     


Thread Information
Users Browsing this Thread

There are currently 1 users browsing this thread. (0 members and 1 guests)


User Tag List

Similar Threads

  1. Replies: 17
    Last Post: 07-26-2018, 09:26 AM
  2. Replies: 16
    Last Post: 01-06-2018, 08:28 AM
  3. Replies: 53
    Last Post: 07-14-2017, 06:01 AM
  4. OSRS Wiki Drop table
    By Lorex in forum Requests
    Replies: 0
    Last Post: 10-10-2016, 07:58 AM
  5. Replies: 3
    Last Post: 07-10-2015, 02:42 AM
Posting Permissions
  • You may not post new threads
  • You may not post replies
  • You may not post attachments
  • You may not edit your posts
  •