Originally Posted by
FullyCharged
only 1 at a time eh...
Concurrent implementation of this, uses multiple threads to load the data faster (although order isn't guaranteed)
Code:
import java.io.BufferedReader;
import java.io.FileWriter;
import java.io.InputStreamReader;
import java.net.URL;
import java.net.URLConnection;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Queue;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
/**
*
* @author http://www.rune-server.org/members/deadpool/ / Jack
* @author lare96
*/
public final class GEPriceDumper {
private static final Queue<String> STRINGS = new ConcurrentLinkedQueue<>();
public static void main(String[] args) {
ItemDef.unpackConfig();
dumpBonuses(13223);
}
public static void dumpBonuses(int totalItems) {
CountDownLatch latch = new CountDownLatch(totalItems + 1);
Path directory = Paths.get("./geprices.txt");
ExecutorService service = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors());
for (int i = 0; i < totalItems; i++) {
ItemDef item = ItemDef.forId(i);
if (item.name.length() <= 0)
continue;
service.execute(new DumpPrice(latch, item));
}
latch.await();
try (FileWriter fw = new FileWriter(directory.toFile())) {
for (;;) {
String next = STRINGS.poll();
if (next == null)
break;
fw.write(next);
}
} catch (Exception e) {
e.printStackTrace();
}
}
private static final class DumpPrice implements Runnable {
private final CountDownLatch latch;
private final ItemDef item;
public DumpPrice(CountDownLatch latch, ItemDef item) {
this.latch = latch;
this.item = item;
}
@Override
public void run() {
try {
URL url = new URL("http://2007.runescape.wikia.com/wiki/" + item.name.replaceAll(" ", "_"));
URLConnection con = url.openConnection();
BufferedReader in = new BufferedReader(new InputStreamReader(con.getInputStream()));
String line;
System.out.println("Connection to " + url + "");
while ((line = in.readLine()) != null) {
try {
if (line.contains("<span id=\"GEPrice\"><span class=\"GEItem\"><span>")) {
line = line.replace("</span></span> coins</span>", "").replace(",", "").replace("</th><td>", "").replace(
"<span id=\"GEPrice\"><span class=\"GEItem\"><span>", "");
if (line.contains("' not found.--></span>"))
continue;
System.out.println("Line: " + line);
STRINGS.add(item.id + " " + line + "\n");
}
} catch (NumberFormatException e) {
}
}
in.close();
} catch (Exception e) {
e.printStackTrace();
} finally {
latch.countDown();
}
}
}
}