InitialCode Commit
This commit is contained in:
Mueller Wayan
@@ -0,0 +1,66 @@
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using MTG_CollectionVerwaltung.API_Connector;
|
||||
using MTG_CollectionVerwaltung.Data.MTGCollection;
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
public class ScryfallDBUpdater : BackgroundService
|
||||
{
|
||||
private readonly ILogger<ScryfallDBUpdater> _logger;
|
||||
private readonly IServiceProvider _serviceProvider;
|
||||
private readonly TimeSpan interval = TimeSpan.FromDays(7);
|
||||
private const string actFile = "nextActualisation.txt";
|
||||
private const string dwlFile = "nextDownload.txt";
|
||||
|
||||
public ScryfallDBUpdater(ILogger<ScryfallDBUpdater> logger, IServiceProvider serviceProvider)
|
||||
{
|
||||
_logger = logger;
|
||||
_serviceProvider = serviceProvider;
|
||||
}
|
||||
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
DateTime nextUpdate = DateTime.MinValue;
|
||||
if (File.Exists(actFile) && DateTime.TryParse(File.ReadAllText(actFile), out var fileTime))
|
||||
{
|
||||
nextUpdate = fileTime;
|
||||
}
|
||||
|
||||
var delay = TimeSpan.FromDays(7) - (DateTime.Now-nextUpdate);
|
||||
if (delay < TimeSpan.Zero) delay = TimeSpan.Zero;
|
||||
bool dwupdate = true;
|
||||
await Task.Delay(delay, stoppingToken);
|
||||
|
||||
while (!stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
try
|
||||
{
|
||||
using var scope = _serviceProvider.CreateScope();
|
||||
var dbContext = scope.ServiceProvider.GetRequiredService<MTGContext>();
|
||||
var connector = new ScryfallConnector(dbContext);
|
||||
_logger.LogInformation("Scryfall DB Update startet...");
|
||||
if (File.Exists(dwlFile) && DateTime.TryParse(File.ReadAllText(dwlFile), out var dwTime))
|
||||
{
|
||||
dwupdate = dwTime.Day != DateTime.Now.Day;
|
||||
}
|
||||
if(dwupdate) await connector.LoadDataAsync();
|
||||
File.WriteAllText(dwlFile, DateTime.Now.ToString());
|
||||
await connector.ActualizeDbAsync();
|
||||
File.WriteAllText(actFile, DateTime.Now.ToString());
|
||||
_logger.LogInformation("Scryfall DB Update abgeschlossen.");
|
||||
dbContext.ChangeTracker.Clear();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Fehler beim Scryfall DB Update");
|
||||
}
|
||||
|
||||
await Task.Delay(interval, stoppingToken);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,394 @@
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.EntityFrameworkCore.Diagnostics;
|
||||
using MTG_CollectionVerwaltung.Data.MTGCollection;
|
||||
using Newtonsoft.Json;
|
||||
using Newtonsoft.Json.Linq;
|
||||
|
||||
|
||||
namespace MTG_CollectionVerwaltung.API_Connector
|
||||
{
|
||||
public class ScryfallConnector
|
||||
{
|
||||
private const string SetFile = "scryfall_Sets.json";
|
||||
private const string BulkDataUrl = "https://api.scryfall.com/bulk-data";
|
||||
private const string CardFile = "scryfall_Cards.json";
|
||||
|
||||
private readonly MTGContext _mtgContext;
|
||||
|
||||
private bool _ftsCreated = false;
|
||||
|
||||
public ScryfallConnector(MTGContext context)
|
||||
{
|
||||
_mtgContext = context;
|
||||
}
|
||||
|
||||
public async Task LoadDataAsync()
|
||||
{
|
||||
var dataUrl = await GetBulkMetaAsync();
|
||||
if (string.IsNullOrEmpty(dataUrl)) return;
|
||||
|
||||
await Task.WhenAll(DownloadDataAsync(dataUrl), DownloadSetDataAsync());
|
||||
}
|
||||
|
||||
public async Task ActualizeDbAsync()
|
||||
{
|
||||
_mtgContext.ChangeTracker.AutoDetectChangesEnabled = false;
|
||||
|
||||
await ProcessSetsAsync();
|
||||
await ProcessCardsAsync();
|
||||
|
||||
await _mtgContext.SaveChangesAsync();
|
||||
_mtgContext.ChangeTracker.AutoDetectChangesEnabled = true;
|
||||
}
|
||||
|
||||
#region Private Methods
|
||||
|
||||
private async Task<string?> GetBulkMetaAsync()
|
||||
{
|
||||
using var client = new HttpClient();
|
||||
try
|
||||
{
|
||||
client.DefaultRequestHeaders.Add("User-Agent", "MTG_HomeServer/1.0");
|
||||
var metaJson = await client.GetStringAsync(BulkDataUrl);
|
||||
var meta = JObject.Parse(metaJson);
|
||||
var dataArray = meta["data"] as JArray;
|
||||
foreach (var entry in dataArray!)
|
||||
{
|
||||
if (entry["type"]?.ToString() == "all_cards")
|
||||
return entry["download_uri"]?.ToString();
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Console.WriteLine($"Error fetching bulk meta: {ex.Message}");
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private async Task DownloadDataAsync(string downloadUrl)
|
||||
{
|
||||
using var client = new HttpClient();
|
||||
Console.WriteLine($"Downloading cards from {downloadUrl}");
|
||||
|
||||
try
|
||||
{
|
||||
client.DefaultRequestHeaders.Add("User-Agent", "MTG_HomeServer/1.0");
|
||||
using var stream = await client.GetStreamAsync(downloadUrl);
|
||||
using var fileStream = File.Create(CardFile);
|
||||
await stream.CopyToAsync(fileStream);
|
||||
Console.WriteLine($"Saved cards to {CardFile}");
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Console.WriteLine($"Failed to download cards: {ex.Message}");
|
||||
}
|
||||
}
|
||||
|
||||
private async Task DownloadSetDataAsync()
|
||||
{
|
||||
using var client = new HttpClient();
|
||||
Console.WriteLine("Downloading sets...");
|
||||
|
||||
try
|
||||
{
|
||||
client.DefaultRequestHeaders.Add("User-Agent", "MTG_HomeServer/1.0");
|
||||
using var stream = await client.GetStreamAsync("https://api.scryfall.com/sets");
|
||||
using var fileStream = File.Create(SetFile);
|
||||
await stream.CopyToAsync(fileStream);
|
||||
Console.WriteLine($"Saved sets to {SetFile}");
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Console.WriteLine($"Failed to download sets: {ex.Message}");
|
||||
}
|
||||
}
|
||||
|
||||
private async Task ProcessSetsAsync()
|
||||
{
|
||||
using var setStream = File.OpenRead(SetFile);
|
||||
using var reader = new StreamReader(setStream);
|
||||
using var json = new JsonTextReader(reader);
|
||||
|
||||
var serializer = new JsonSerializer();
|
||||
int counter = 0;
|
||||
|
||||
while (await json.ReadAsync())
|
||||
{
|
||||
if (json.TokenType == JsonToken.PropertyName && (string)json.Value == "data")
|
||||
{
|
||||
await json.ReadAsync(); // move to StartArray
|
||||
|
||||
if (json.TokenType != JsonToken.StartArray)
|
||||
throw new InvalidDataException("Expected 'data' to be an array.");
|
||||
|
||||
// Array durchlaufen
|
||||
while (await json.ReadAsync() && json.TokenType != JsonToken.EndArray)
|
||||
{
|
||||
if (json.TokenType != JsonToken.StartObject) continue;
|
||||
|
||||
var setJson = serializer.Deserialize<Dictionary<string, object>>(json);
|
||||
if (setJson == null) continue;
|
||||
|
||||
string setCode = setJson["code"]?.ToString();
|
||||
|
||||
if (setCode == null) continue;
|
||||
|
||||
var set = await _mtgContext.Sets.FindAsync(setCode);
|
||||
if (set == null)
|
||||
{
|
||||
set = new Set
|
||||
{
|
||||
SetCode = setCode,
|
||||
SetName = setJson["name"]?.ToString(),
|
||||
CardCount = Convert.ToInt32(setJson.GetValueOrDefault("card_count") ?? 0),
|
||||
SetType = setJson["set_type"]?.ToString(),
|
||||
SetIcon = setJson["icon_svg_uri"]?.ToString(),
|
||||
ReleaseDate = (DateTime)(DateTime.TryParse(setJson["released_at"]?.ToString(), out var dt) ? dt : (DateTime?)null)
|
||||
};
|
||||
_mtgContext.Sets.Add(set);
|
||||
}
|
||||
|
||||
counter++;
|
||||
if (counter >= 500)
|
||||
{
|
||||
counter = 0;
|
||||
await _mtgContext.SaveChangesAsync();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
await _mtgContext.SaveChangesAsync();
|
||||
}
|
||||
|
||||
private async Task ProcessCardsAsync()
|
||||
{
|
||||
// --- Caches ---
|
||||
Dictionary<string, Supertype> superTypeCache = _mtgContext.Supertypes.ToDictionary(s => s.TypeEn);
|
||||
Dictionary<string, MTGType> typeCache = _mtgContext.Types.ToDictionary(t => t.TypeEn);
|
||||
Dictionary<string, Set> setCache = _mtgContext.Sets.ToDictionary(s => s.SetCode);
|
||||
|
||||
using var fileStream = File.OpenRead(CardFile);
|
||||
using var reader = new StreamReader(fileStream);
|
||||
using var json = new JsonTextReader(reader);
|
||||
|
||||
var serializer = new JsonSerializer();
|
||||
int counter = 0;
|
||||
|
||||
while (await json.ReadAsync())
|
||||
{
|
||||
if (json.TokenType != JsonToken.StartObject) continue;
|
||||
|
||||
var cardJson = serializer.Deserialize<Dictionary<string, object>>(json);
|
||||
if (cardJson == null) continue;
|
||||
|
||||
try
|
||||
{
|
||||
|
||||
if (cardJson.Keys.Contains("card_faces"))
|
||||
{
|
||||
if (cardJson.TryGetValue("card_faces", out var cardFaces) && cardFaces is JArray dataArray)
|
||||
{
|
||||
string cardId = cardJson["id"].ToString();
|
||||
int faceCounter = 0;
|
||||
foreach (JObject face in dataArray)
|
||||
{
|
||||
cardJson["id"] = cardId + faceCounter;
|
||||
cardJson["type_line"] = face["type_line"];
|
||||
cardJson["oracle_text"] = face["oracle_text"];
|
||||
if (face.TryGetValue("flavor_text", out var flavText)) cardJson["flavor_text"] = flavText;
|
||||
cardJson["name"] = face["name"];
|
||||
cardJson["mana_cost"] = face["mana_cost"];
|
||||
if(faceCounter == dataArray.Count - 1)
|
||||
{
|
||||
cardJson["otherFace"] = cardId + 0;
|
||||
}
|
||||
else
|
||||
{
|
||||
cardJson["otherFace"] = cardId + faceCounter + 1;
|
||||
}
|
||||
string cardLang = cardJson["lang"]?.ToString();
|
||||
if (cardLang == null) return;
|
||||
if (cardLang != "en" && cardLang != "de") continue;
|
||||
if (cardLang == "de") {
|
||||
cardJson["printed_type_line"] = face["printed_type_line"];
|
||||
cardJson["printed_text"] = face["printed_text"];
|
||||
cardJson["printed_name"] = face["printed_name"];
|
||||
}
|
||||
await addCard(cardJson, superTypeCache, typeCache, setCache);
|
||||
faceCounter++;
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
await addCard(cardJson,superTypeCache,typeCache,setCache);
|
||||
}
|
||||
|
||||
|
||||
|
||||
counter++;
|
||||
if (counter >= 500)
|
||||
{
|
||||
counter = 0;
|
||||
await _mtgContext.SaveChangesAsync();
|
||||
if (!_ftsCreated)
|
||||
{
|
||||
_mtgContext.EnsureFtsInitialized();
|
||||
_ftsCreated = true;
|
||||
}
|
||||
//_mtgContext.ChangeTracker.Clear();
|
||||
}
|
||||
}
|
||||
catch(Exception ex)
|
||||
{
|
||||
throw new Exception("Error At Card: " + JsonConvert.SerializeObject(cardJson,Formatting.Indented) ,ex);
|
||||
}
|
||||
}
|
||||
|
||||
await _mtgContext.SaveChangesAsync();
|
||||
_mtgContext.RebuildCardPrintingsFts();
|
||||
}
|
||||
|
||||
private async Task addCard(Dictionary<string, object> cardJson, Dictionary<string, Supertype> superTypeCache, Dictionary<string, MTGType> typeCache, Dictionary<string, Set> setCache)
|
||||
{
|
||||
string cardLang = cardJson["lang"]?.ToString();
|
||||
if (cardLang == null) return;
|
||||
if (cardLang != "en" && cardLang != "de") return;
|
||||
|
||||
string typeline = cardJson["type_line"]?.ToString() ?? "";
|
||||
string printTypeline = cardLang == "de" ? cardJson.TryGetValue("printed_type_line", out object? value) ? value?.ToString() : cardJson["type_line"].ToString() : null;
|
||||
string cardName = cardJson["name"]?.ToString() ?? "";
|
||||
string setCode = cardJson["set"]?.ToString() ?? "";
|
||||
string collectorNumber = cardJson["collector_number"]?.ToString();
|
||||
if (collectorNumber == null) return;
|
||||
|
||||
if (!setCache.TryGetValue(setCode, out var set)) return;
|
||||
|
||||
// --- Supertypen ---
|
||||
string sSuperType = typeline.Split('—')[0].Trim();
|
||||
string? printedSuperType = cardLang == "de" && printTypeline != null ? printTypeline.Split('—')[0].Trim() : "";
|
||||
|
||||
if (!superTypeCache.TryGetValue(sSuperType, out var superType))
|
||||
{
|
||||
superType = new Supertype { TypeEn = sSuperType, TypeDe = printedSuperType };
|
||||
_mtgContext.Supertypes.Add(superType);
|
||||
superTypeCache[sSuperType] = superType;
|
||||
}
|
||||
else
|
||||
{
|
||||
if (superType.TypeDe == null || superType.TypeDe == string.Empty)
|
||||
superType.TypeDe = printedSuperType;
|
||||
}
|
||||
|
||||
|
||||
// --- Typen ---
|
||||
var types = typeline.Split('—');
|
||||
if (types.Length > 1)
|
||||
{
|
||||
types = types[1].Trim().Split(' ', StringSplitOptions.RemoveEmptyEntries);
|
||||
}
|
||||
var printedTypes = cardLang == "de" && printTypeline != null ? printTypeline.Split('—') : null;
|
||||
if (printedTypes != null && printedTypes.Length > 1)
|
||||
printedTypes = printedTypes[1].Trim().Split(' ');
|
||||
|
||||
var mtgTypes = new List<MTGType>();
|
||||
for (int i = 0; i < types.Length; i++)
|
||||
{
|
||||
string typeName = types[i];
|
||||
if (!typeCache.TryGetValue(typeName, out var mtgType))
|
||||
{
|
||||
mtgType = new MTGType
|
||||
{
|
||||
TypeEn = typeName,
|
||||
TypeDe = cardLang == "de" && printedTypes != null && i < printedTypes.Length ? printedTypes[i] : ""
|
||||
};
|
||||
_mtgContext.Types.Add(mtgType);
|
||||
typeCache[typeName] = mtgType;
|
||||
}
|
||||
else
|
||||
{
|
||||
if (mtgType.TypeDe == null || mtgType.TypeDe == string.Empty)
|
||||
mtgType.TypeDe = cardLang == "de" && printedTypes != null && i < printedTypes.Length ? printedTypes[i] : "";
|
||||
}
|
||||
mtgTypes.Add(mtgType);
|
||||
}
|
||||
|
||||
// --- Karte identifizieren ---
|
||||
var card = await _mtgContext.Cards
|
||||
.Include(c => c.Printings)
|
||||
.FirstOrDefaultAsync(c => c.CollectorNumber == collectorNumber && c.SetCode == setCode);
|
||||
|
||||
var cardPrinting = new CardPrinting
|
||||
{
|
||||
CardId = cardJson["id"]?.ToString(),
|
||||
Lang = cardLang,
|
||||
Name = cardLang == "de" && cardJson.ContainsKey("printed_name") && cardJson["printed_name"] != null ? cardJson["printed_name"]?.ToString() : cardJson["name"]?.ToString(),
|
||||
OracleText = cardLang == "de" && cardJson.ContainsKey("printed_text") ? cardJson.GetValueOrDefault("printed_text")?.ToString() : cardJson.GetValueOrDefault("oracle_text")?.ToString(),
|
||||
FlavorText = cardJson.GetValueOrDefault("flavor_text")?.ToString(),
|
||||
ImageUriSmall = (cardJson.GetValueOrDefault("image_uris") as JObject)?["small"]?.ToString(),
|
||||
ImageUriNormal = (cardJson.GetValueOrDefault("image_uris") as JObject)?["normal"]?.ToString(),
|
||||
ImageUriLarge = (cardJson.GetValueOrDefault("image_uris") as JObject)?["large"]?.ToString()
|
||||
};
|
||||
|
||||
if (card == null)
|
||||
{
|
||||
// Neue Karte anlegen
|
||||
card = new Card
|
||||
{
|
||||
Id = cardJson["id"]?.ToString(),
|
||||
//CardName = cardName,
|
||||
//SetCode = setCode,
|
||||
SuperTypeId = superType.TypeId,
|
||||
CollectorNumber = collectorNumber,
|
||||
ManaCost = cardJson.GetValueOrDefault("mana_cost")?.ToString(),
|
||||
Foil = Convert.ToBoolean(cardJson.GetValueOrDefault("foil") ?? false),
|
||||
NonFoil = Convert.ToBoolean(cardJson.GetValueOrDefault("nonfoil") ?? false),
|
||||
Rarity = cardJson["rarity"]?.ToString(),
|
||||
Power = cardJson.GetValueOrDefault("power")?.ToString(),
|
||||
Toughness = cardJson.GetValueOrDefault("toughness")?.ToString(),
|
||||
Cmc = Convert.ToInt32(cardJson.GetValueOrDefault("cmc") ?? 0),
|
||||
OtherFace = cardJson.GetValueOrDefault("otherFace")?.ToString(),
|
||||
SuperType = superType,
|
||||
Set = set,
|
||||
Printings = new List<CardPrinting> { cardPrinting }
|
||||
};
|
||||
_mtgContext.Cards.Add(card);
|
||||
|
||||
foreach (var mtgType in mtgTypes)
|
||||
_mtgContext.CardTypes.Add(new CardType { Card = card, Type = mtgType });
|
||||
}
|
||||
else
|
||||
{
|
||||
// Prüfen, ob Printing für diese Sprache existiert
|
||||
var existingPrinting = card.Printings.FirstOrDefault(p => p.Lang == cardLang);
|
||||
if (existingPrinting == null)
|
||||
{
|
||||
card.Printings.Add(cardPrinting);
|
||||
|
||||
}
|
||||
else
|
||||
{
|
||||
// Update vorhandenes Printing, falls sich etwas geändert hat
|
||||
bool changed = false;
|
||||
|
||||
if (existingPrinting.Name != cardPrinting.Name) { existingPrinting.Name = cardPrinting.Name; changed = true; }
|
||||
if (existingPrinting.OracleText != cardPrinting.OracleText) { existingPrinting.OracleText = cardPrinting.OracleText; changed = true; }
|
||||
if (existingPrinting.FlavorText != cardPrinting.FlavorText) { existingPrinting.FlavorText = cardPrinting.FlavorText; changed = true; }
|
||||
if (existingPrinting.ImageUriSmall != cardPrinting.ImageUriSmall) { existingPrinting.ImageUriSmall = cardPrinting.ImageUriSmall; changed = true; }
|
||||
if (existingPrinting.ImageUriNormal != cardPrinting.ImageUriNormal) { existingPrinting.ImageUriNormal = cardPrinting.ImageUriNormal; changed = true; }
|
||||
if (existingPrinting.ImageUriLarge != cardPrinting.ImageUriLarge) { existingPrinting.ImageUriLarge = cardPrinting.ImageUriLarge; changed = true; }
|
||||
|
||||
if (changed)
|
||||
{
|
||||
//_mtgContext.CardPrintings.Update(existingPrinting);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
#endregion
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user