ambidextrous/gemp-lotr
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Extracting provider for card prices.

Browse Source
This commit is contained in:
MarcinSc
2016-01-04 15:29:19 -08:00
parent 29f7bfa9e5
commit 37e79d81c8
12 changed files with 371 additions and 148 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
gemp-lotr/gemp-lotr-async/src/main/java/com/gempukku/lotro/async/handler/MtgCardsRequestHandler.java
View File

@@ -5,9 +5,11 @@ import com.gempukku.mtg.MtgCardServer;
import org.jboss.netty.channel.MessageEvent;
import org.jboss.netty.handler.codec.http.HttpHeaders;
import org.jboss.netty.handler.codec.http.HttpRequest;
import org.jboss.netty.handler.codec.http.QueryStringDecoder;
import java.lang.reflect.Type;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static org.jboss.netty.handler.codec.http.HttpHeaders.Names.CONTENT_TYPE;
@@ -22,7 +24,10 @@ public class MtgCardsRequestHandler implements UriRequestHandler {
@Override
public void handleRequest(String uri, HttpRequest request, Map<Type, Object> context, ResponseWriter responseWriter, MessageEvent e) throws Exception {
MtgCardServer.CardDatabaseHolder cardDatabaseHolder = _mtgCardServer.getCardDatabaseHolder();
QueryStringDecoder queryDecoder = new QueryStringDecoder(request.getUri());
String provider = getQueryParameterSafely(queryDecoder, "provider");
MtgCardServer.CardDatabaseHolder cardDatabaseHolder = _mtgCardServer.getCardDatabaseHolder(provider);
if (cardDatabaseHolder == null || clientHasCurrentVersion(request, cardDatabaseHolder.getUpdateMarker())) {
responseWriter.writeError(304);
return;
@@ -46,4 +51,12 @@ public class MtgCardsRequestHandler implements UriRequestHandler {
}
return false;
}
protected String getQueryParameterSafely(QueryStringDecoder queryStringDecoder, String parameterName) {
List<String> parameterValues = queryStringDecoder.getParameters().get(parameterName);
if (parameterValues != null && parameterValues.size() > 0)
return parameterValues.get(0);
else
return null;
}
}

14
gemp-lotr/gemp-lotr-server/src/main/java/com/gempukku/lotro/builder/ServerBuilder.java
View File

@@ -35,10 +35,7 @@ import com.gempukku.lotro.tournament.TournamentService;
import com.gempukku.mtg.MtgCardServer;
import java.lang.reflect.Type;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Map;
import java.util.TimeZone;
public class ServerBuilder {
public static void fillObjectMap(Map<Type, Object> objectMap) {
@@ -125,16 +122,7 @@ public class ServerBuilder {
extract(objectMap, CardSets.class)
));
try {
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
sdf.setTimeZone(TimeZone.getTimeZone("EST"));
long someDownloadTime = sdf.parse("2015-01-01 04:25:00").getTime();
objectMap.put(MtgCardServer.class,
new MtgCardServer(someDownloadTime, 24L * 60 * 60 * 1000));
} catch (ParseException exp) {
}
objectMap.put(MtgCardServer.class, new MtgCardServer());
}
private static <T> T extract(Map<Type, Object> objectMap, Class<T> clazz) {

25
gemp-lotr/gemp-lotr-server/src/main/java/com/gempukku/mtg/CardData.java Normal file
View File

@@ -0,0 +1,25 @@
package com.gempukku.mtg;
public class CardData {
private final String _id;
private final String _name;
private final int _price;
public CardData(String id, String name, int price) {
_id = id;
_name = name;
_price = price;
}
public String getId() {
return _id;
}
public String getName() {
return _name;
}
public int getPrice() {
return _price;
}
}

179
gemp-lotr/gemp-lotr-server/src/main/java/com/gempukku/mtg/MtgCardServer.java
View File

@@ -1,54 +1,70 @@
package com.gempukku.mtg;
import com.gempukku.lotro.AbstractServer;
import org.apache.log4j.Logger;
import com.gempukku.mtg.provider.mtggoldfish.MtgGoldfishProvider;
import org.json.simple.JSONArray;
import org.json.simple.JSONObject;
import org.jsoup.HttpStatusException;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import java.io.IOException;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.net.URL;
import java.nio.charset.Charset;
import java.util.*;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
public class MtgCardServer extends AbstractServer {
private static final Logger LOG = Logger.getLogger(MtgCardServer.class);
private Map<String, MtgDataProvider> _dataProviders = new HashMap<String, MtgDataProvider>();
private volatile CardDatabaseHolder _cardDatabaseHolder;
private long _downloadEvery;
private long _nextStart;
private Map<String, CardDatabaseHolder> _providersData = Collections.synchronizedMap(new HashMap<String, CardDatabaseHolder>());
public MtgCardServer(long someDownloadTime, long downloadEvery) {
_downloadEvery = downloadEvery;
long number = (System.currentTimeMillis() - someDownloadTime) / downloadEvery;
_nextStart = someDownloadTime + number * downloadEvery;
public MtgCardServer() {
_dataProviders.put("mtgGoldFish", new MtgGoldfishProvider());
}
private void updateDatabase() {
Thread thr = new Thread(new UpdateDatabase());
thr.start();
}
public CardDatabaseHolder getCardDatabaseHolder() {
return _cardDatabaseHolder;
public CardDatabaseHolder getCardDatabaseHolder(String providerName) {
return _providersData.get(providerName);
}
@Override
protected void cleanup() {
if (_nextStart < System.currentTimeMillis()) {
_nextStart += _downloadEvery;
updateDatabase();
for (Map.Entry<String, MtgDataProvider> dataProviderEntry : _dataProviders.entrySet()) {
final String key = dataProviderEntry.getKey();
MtgDataProvider dataProvider = dataProviderEntry.getValue();
if (dataProvider.shouldBeUpdated()) {
dataProvider.update(
new MtgDataProvider.UpdateCallback() {
@Override
public void callback(TimestampedCardCollection result) {
_providersData.put(key, new CardDatabaseHolder(marshallData(result.getSetCardDataList()), result.getUpdateMarker()));
}
});
}
}
}
public static byte[] marshallData(List<SetCardData> setCardDataList) {
JSONArray allSets = new JSONArray();
for (SetCardData setCardData : setCardDataList) {
List<Object> setArray = new LinkedList<Object>();
for (CardData cardData : setCardData.getAllCards()) {
Map<String, Object> cardObject = new LinkedHashMap<String, Object>();
cardObject.put("id", cardData.getId());
cardObject.put("name", cardData.getName());
cardObject.put("price", cardData.getPrice());
setArray.add(cardObject);
}
Map<String, Object> setObject = new LinkedHashMap<String, Object>();
setObject.put("name", setCardData.getSetName());
setObject.put("cards", setArray);
allSets.add(setObject);
}
return allSets.toJSONString().getBytes(Charset.forName("UTF-8"));
}
public class CardDatabaseHolder {
private String _updateMarker;
private byte[] _bytes;
@@ -66,103 +82,4 @@ public class MtgCardServer extends AbstractServer {
return _bytes;
}
}
private class UpdateDatabase implements Runnable {
private static final int SLEEP_MINIMUM = 8 * 1000;
private static final int SLEEP_MAXIMUM = 12 * 1000;
private static final int TIMEOUT = 5 * 60 * 1000;
public void run() {
String updateMarker = String.valueOf(System.currentTimeMillis());
try {
JSONArray resultArray = new JSONArray();
List<MtgCardSet> mtgCardSets = downloadSetList();
for (MtgCardSet mtgCardSet : mtgCardSets) {
downloadSet(mtgCardSet, resultArray);
}
String resultJson = resultArray.toJSONString();
_cardDatabaseHolder = new CardDatabaseHolder(resultJson.getBytes("UTF-8"), updateMarker);
} catch (Exception exp) {
LOG.error("Error downloading card database", exp);
// Temporarily we sent the error as response
StringWriter writer = new StringWriter();
exp.printStackTrace(new PrintWriter(writer));
_cardDatabaseHolder = new CardDatabaseHolder(writer.toString().getBytes(Charset.defaultCharset()), updateMarker);
}
}
private List<MtgCardSet> downloadSetList() throws IOException {
List<MtgCardSet> results = new LinkedList<MtgCardSet>();
Document doc = Jsoup.connect("http://www.mtggoldfish.com/prices/paper/standard").get();
String[] priceListTypes = new String[]{"Standard", "Modern", "Legacy", "Special"};
for (String priceListType : priceListTypes) {
Elements cardElements = doc.select(".priceList-setMenu-" + priceListType);
Elements setImages = cardElements.select("li img");
for (Element setImage : setImages) {
String uriPostfix = setImage.attr("alt");
String setName = setImage.parent().text();
results.add(new MtgCardSet(uriPostfix, setName));
}
}
return results;
}
private void downloadSet(MtgCardSet mtgCardSet, JSONArray jsonArray) throws IOException {
try {
Thread.sleep(getSleepTime());
} catch (InterruptedException e) {
}
try {
Document doc = Jsoup.parse(new URL("http://www.mtggoldfish.com/index/" + mtgCardSet.getUrlPostfix()), TIMEOUT);
boolean isInPaper = (doc.select("#priceHistoryTabs [href=#tab-paper]").size() > 0);
if (isInPaper) {
JSONArray setArray = new JSONArray();
Map<String, Object> setObject = new LinkedHashMap<String, Object>();
setObject.put("name", mtgCardSet.getInfoLine());
setObject.put("cards", setArray);
jsonArray.add(setObject);
Elements cardElements = doc.select(".index-price-table-paper tbody tr");
for (Element cardElement : cardElements) {
String cardName = cardElement.select("td:nth-of-type(1) a").text();
String cardId = mtgCardSet.getUrlPostfix() + "-" + cardName;
float price = Float.parseFloat(cardElement.select("td:nth-of-type(4)").text().replace(",", ""));
JSONObject cardObject = new JSONObject();
cardObject.put("id", cardId);
cardObject.put("name", cardName);
cardObject.put("price", Math.round(price * 100));
setArray.add(cardObject);
}
System.out.println("Finished set " + mtgCardSet.getInfoLine());
boolean hasFoils = doc.select(".index-price-header-foil-switcher").size() > 0;
if (hasFoils) {
downloadSet(new MtgCardSet(mtgCardSet.getUrlPostfix() + "_F", mtgCardSet.getInfoLine() + " · Foil"), jsonArray);
}
}
} catch (HttpStatusException exp) {
}
}
private int getSleepTime() {
Random rnd = new Random();
return SLEEP_MINIMUM + rnd.nextInt(SLEEP_MAXIMUM - SLEEP_MINIMUM);
}
}
public static void main(String[] args) {
MtgCardServer mtgCardServer = new MtgCardServer(0, 1);
UpdateDatabase updateDatabase = mtgCardServer.new UpdateDatabase();
updateDatabase.run();
System.out.println(new String(mtgCardServer._cardDatabaseHolder._bytes));
}
}

11
gemp-lotr/gemp-lotr-server/src/main/java/com/gempukku/mtg/MtgDataProvider.java Normal file
View File

@@ -0,0 +1,11 @@
package com.gempukku.mtg;
public interface MtgDataProvider {
boolean shouldBeUpdated();
void update(UpdateCallback updateCallback);
interface UpdateCallback {
void callback(TimestampedCardCollection result);
}
}

21
gemp-lotr/gemp-lotr-server/src/main/java/com/gempukku/mtg/SetCardData.java Normal file
View File

@@ -0,0 +1,21 @@
package com.gempukku.mtg;
import java.util.List;
public class SetCardData {
private String _setName;
private List<CardData> _allCards;
public SetCardData(String setName, List<CardData> allCards) {
_setName = setName;
_allCards = allCards;
}
public List<CardData> getAllCards() {
return _allCards;
}
public String getSetName() {
return _setName;
}
}

21
gemp-lotr/gemp-lotr-server/src/main/java/com/gempukku/mtg/TimestampedCardCollection.java Normal file
View File

@@ -0,0 +1,21 @@
package com.gempukku.mtg;
import java.util.List;
public class TimestampedCardCollection {
private final String _updateMarker;
private final List<SetCardData> _setCardDataList;
public TimestampedCardCollection(String updateMarker, List<SetCardData> setCardDataList) {
_updateMarker = updateMarker;
_setCardDataList = setCardDataList;
}
public List<SetCardData> getSetCardDataList() {
return _setCardDataList;
}
public String getUpdateMarker() {
return _updateMarker;
}
}

22
gemp-lotr/gemp-lotr-server/src/main/java/com/gempukku/mtg/provider/RetryUtil.java Normal file
View File

@@ -0,0 +1,22 @@
package com.gempukku.mtg.provider;
public class RetryUtil {
public static <T, U extends Exception> T executeInRetry(Retryable<T, U> task, int retries) throws U {
int retryCount = 0;
while (true) {
try {
return task.execute();
} catch (Exception exp) {
if (task.isRetryable(exp)) {
retryCount++;
if (retryCount == retries)
throw (U) exp;
} else if (exp instanceof RuntimeException) {
throw (RuntimeException) exp;
} else {
throw new RuntimeException(exp);
}
}
}
}
}

7
gemp-lotr/gemp-lotr-server/src/main/java/com/gempukku/mtg/provider/Retryable.java Normal file
View File

@@ -0,0 +1,7 @@
package com.gempukku.mtg.provider;
public interface Retryable<T, U extends Exception> {
T execute() throws U;
boolean isRetryable(Exception exception);
}

6
gemp-lotr/gemp-lotr-server/src/main/java/com/gempukku/mtg/MtgCardSet.java → gemp-lotr/gemp-lotr-server/src/main/java/com/gempukku/mtg/provider/mtggoldfish/MtgGoldfishCardSet.java
View File

@@ -1,10 +1,10 @@
package com.gempukku.mtg;
package com.gempukku.mtg.provider.mtggoldfish;
public class MtgCardSet {
public class MtgGoldfishCardSet {
private String _urlPostfix;
private String _infoLine;
public MtgCardSet(String urlPostfix, String infoLine) {
public MtgGoldfishCardSet(String urlPostfix, String infoLine) {
_urlPostfix = urlPostfix;
_infoLine = infoLine;
}

176
gemp-lotr/gemp-lotr-server/src/main/java/com/gempukku/mtg/provider/mtggoldfish/MtgGoldfishProvider.java Normal file
View File

@@ -0,0 +1,176 @@
package com.gempukku.mtg.provider.mtggoldfish;
import com.gempukku.mtg.CardData;
import com.gempukku.mtg.MtgCardServer;
import com.gempukku.mtg.MtgDataProvider;
import com.gempukku.mtg.SetCardData;
import com.gempukku.mtg.TimestampedCardCollection;
import com.gempukku.mtg.provider.RetryUtil;
import com.gempukku.mtg.provider.Retryable;
import org.apache.log4j.Logger;
import org.jsoup.HttpStatusException;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import java.io.IOException;
import java.net.URL;
import java.nio.charset.Charset;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.LinkedList;
import java.util.List;
import java.util.Random;
import java.util.TimeZone;
public class MtgGoldfishProvider implements MtgDataProvider {
private static final Logger LOG = Logger.getLogger(MtgGoldfishProvider.class);
private static final long DOWNLOAD_EVERY = 24L * 60 * 60 * 1000;
private static final int SLEEP_MINIMUM = 8 * 1000;
private static final int SLEEP_MAXIMUM = 12 * 1000;
private static final int TIMEOUT = 5 * 60 * 1000;
private static final int RETRY_MAX = 5;
private long _nextStart;
public MtgGoldfishProvider() {
try {
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
sdf.setTimeZone(TimeZone.getTimeZone("EST"));
long someDownloadTime = sdf.parse("2015-01-01 04:25:00").getTime();
long number = (System.currentTimeMillis() - someDownloadTime) / DOWNLOAD_EVERY;
_nextStart = someDownloadTime + number * DOWNLOAD_EVERY;
} catch (ParseException exp) {
// ignore
}
}
@Override
public boolean shouldBeUpdated() {
return _nextStart < System.currentTimeMillis();
}
@Override
public void update(final UpdateCallback updateCallback) {
_nextStart += DOWNLOAD_EVERY;
Thread thr = new Thread(
new Runnable() {
@Override
public void run() {
updateCallback.callback(updateDatabase());
}
});
thr.start();
}
private TimestampedCardCollection updateDatabase() {
String updateMarker = String.valueOf(System.currentTimeMillis());
final List<SetCardData> resultArray = new LinkedList<SetCardData>();
try {
List<MtgGoldfishCardSet> mtgCardSets = downloadSetListWithRetry();
for (final MtgGoldfishCardSet mtgCardSet : mtgCardSets) {
downloadSetWithRetry(resultArray, mtgCardSet);
}
} catch (IOException exp) {
LOG.error("Unable to download card list", exp);
}
return new TimestampedCardCollection(updateMarker, resultArray);
}
private List<MtgGoldfishCardSet> downloadSetListWithRetry() throws IOException {
return RetryUtil.executeInRetry(
new Retryable<List<MtgGoldfishCardSet>, IOException>() {
@Override
public List<MtgGoldfishCardSet> execute() throws IOException {
return downloadSetList();
}
@Override
public boolean isRetryable(Exception exception) {
return exception instanceof IOException;
}
}, RETRY_MAX);
}
private List<MtgGoldfishCardSet> downloadSetList() throws IOException {
List<MtgGoldfishCardSet> results = new LinkedList<MtgGoldfishCardSet>();
Document doc = Jsoup.connect("http://www.mtggoldfish.com/prices/paper/standard").get();
String[] priceListTypes = new String[]{"Standard", "Modern", "Legacy", "Special"};
for (String priceListType : priceListTypes) {
Elements cardElements = doc.select(".priceList-setMenu-" + priceListType);
Elements setImages = cardElements.select("li img");
for (Element setImage : setImages) {
String uriPostfix = setImage.attr("alt");
String setName = setImage.parent().text();
results.add(new MtgGoldfishCardSet(uriPostfix, setName));
}
}
return results;
}
private void downloadSetWithRetry(final List<SetCardData> resultArray, final MtgGoldfishCardSet mtgCardSet) throws IOException {
try {
RetryUtil.executeInRetry(
new Retryable<Void, IOException>() {
@Override
public Void execute() throws IOException {
downloadSet(mtgCardSet, resultArray);
return null;
}
@Override
public boolean isRetryable(Exception exception) {
return exception instanceof IOException;
}
}, RETRY_MAX);
} catch (HttpStatusException exp) {
// Unable to download the set - skip it
}
}
private void downloadSet(MtgGoldfishCardSet mtgCardSet, List<SetCardData> result) throws IOException {
try {
Thread.sleep(getSleepTime());
} catch (InterruptedException e) {
// Ignore
}
Document doc = Jsoup.parse(new URL("http://www.mtggoldfish.com/index/" + mtgCardSet.getUrlPostfix()), TIMEOUT);
boolean isInPaper = (doc.select("#priceHistoryTabs [href=#tab-paper]").size() > 0);
if (isInPaper) {
List<CardData> allCards = new LinkedList<CardData>();
Elements cardElements = doc.select(".index-price-table-paper tbody tr");
for (Element cardElement : cardElements) {
String cardName = cardElement.select("td:nth-of-type(1) a").text();
String cardId = mtgCardSet.getUrlPostfix() + "-" + cardName;
float price = Float.parseFloat(cardElement.select("td:nth-of-type(4)").text().replace(",", ""));
allCards.add(new CardData(cardId, cardName, Math.round(price * 100)));
}
boolean hasFoils = doc.select(".index-price-header-foil-switcher").size() > 0;
if (hasFoils) {
downloadSet(new MtgGoldfishCardSet(mtgCardSet.getUrlPostfix() + "_F", mtgCardSet.getInfoLine() + " · Foil"), result);
}
result.add(new SetCardData(mtgCardSet.getInfoLine(), allCards));
}
}
private int getSleepTime() {
Random rnd = new Random();
return SLEEP_MINIMUM + rnd.nextInt(SLEEP_MAXIMUM - SLEEP_MINIMUM);
}
public static void main(String[] args) {
MtgGoldfishProvider mtgCardServer = new MtgGoldfishProvider();
TimestampedCardCollection timestampedCardCollection = mtgCardServer.updateDatabase();
System.out.println(new String(MtgCardServer.marshallData(timestampedCardCollection.getSetCardDataList()), Charset.forName("UTF-8")));
}
}

22
gemp-lotr/gemp-lotr-server/src/main/java/com/gempukku/mtg/provider/tcgplayer/TCGPlayerProvider.java Normal file
View File

@@ -0,0 +1,22 @@
package com.gempukku.mtg.provider.tcgplayer;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import java.io.IOException;
import java.util.LinkedList;
import java.util.List;
public class TCGPlayerProvider {
private List<String> downloadSetList() throws IOException {
List<String> results = new LinkedList<String>();
Document doc = Jsoup.connect("http://shop.tcgplayer.com/magic").get();
for (Element element : doc.select("#SetName option")) {
String setName = element.attr("value");
if (!setName.equals("All"))
results.add(setName);
}
return results;
}
}