using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Diagnostics; using MTG_CollectionVerwaltung.Data.MTGCollection; using Newtonsoft.Json; using Newtonsoft.Json.Linq; namespace MTG_CollectionVerwaltung.API_Connector { public class ScryfallConnector { private const string SetFile = "scryfall_Sets.json"; private const string BulkDataUrl = "https://api.scryfall.com/bulk-data"; private const string CardFile = "scryfall_Cards.json"; private readonly MTGContext _mtgContext; private bool _ftsCreated = false; public ScryfallConnector(MTGContext context) { _mtgContext = context; } public async Task LoadDataAsync() { var dataUrl = await GetBulkMetaAsync(); if (string.IsNullOrEmpty(dataUrl)) return; await Task.WhenAll(DownloadDataAsync(dataUrl), DownloadSetDataAsync()); } public async Task ActualizeDbAsync() { _mtgContext.ChangeTracker.AutoDetectChangesEnabled = false; await ProcessSetsAsync(); await ProcessCardsAsync(); await _mtgContext.SaveChangesAsync(); _mtgContext.ChangeTracker.AutoDetectChangesEnabled = true; } #region Private Methods private async Task GetBulkMetaAsync() { using var client = new HttpClient(); try { client.DefaultRequestHeaders.Add("User-Agent", "MTG_HomeServer/1.0"); var metaJson = await client.GetStringAsync(BulkDataUrl); var meta = JObject.Parse(metaJson); var dataArray = meta["data"] as JArray; foreach (var entry in dataArray!) { if (entry["type"]?.ToString() == "all_cards") return entry["download_uri"]?.ToString(); } } catch (Exception ex) { Console.WriteLine($"Error fetching bulk meta: {ex.Message}"); } return null; } private async Task DownloadDataAsync(string downloadUrl) { using var client = new HttpClient(); Console.WriteLine($"Downloading cards from {downloadUrl}"); try { client.DefaultRequestHeaders.Add("User-Agent", "MTG_HomeServer/1.0"); using var stream = await client.GetStreamAsync(downloadUrl); using var fileStream = File.Create(CardFile); await stream.CopyToAsync(fileStream); Console.WriteLine($"Saved cards to {CardFile}"); } catch (Exception ex) { Console.WriteLine($"Failed to download cards: {ex.Message}"); } } private async Task DownloadSetDataAsync() { using var client = new HttpClient(); Console.WriteLine("Downloading sets..."); try { client.DefaultRequestHeaders.Add("User-Agent", "MTG_HomeServer/1.0"); using var stream = await client.GetStreamAsync("https://api.scryfall.com/sets"); using var fileStream = File.Create(SetFile); await stream.CopyToAsync(fileStream); Console.WriteLine($"Saved sets to {SetFile}"); } catch (Exception ex) { Console.WriteLine($"Failed to download sets: {ex.Message}"); } } private async Task ProcessSetsAsync() { using var setStream = File.OpenRead(SetFile); using var reader = new StreamReader(setStream); using var json = new JsonTextReader(reader); var serializer = new JsonSerializer(); int counter = 0; while (await json.ReadAsync()) { if (json.TokenType == JsonToken.PropertyName && (string)json.Value == "data") { await json.ReadAsync(); // move to StartArray if (json.TokenType != JsonToken.StartArray) throw new InvalidDataException("Expected 'data' to be an array."); // Array durchlaufen while (await json.ReadAsync() && json.TokenType != JsonToken.EndArray) { if (json.TokenType != JsonToken.StartObject) continue; var setJson = serializer.Deserialize>(json); if (setJson == null) continue; string setCode = setJson["code"]?.ToString(); if (setCode == null) continue; var set = await _mtgContext.Sets.FindAsync(setCode); if (set == null) { set = new Set { SetCode = setCode, SetName = setJson["name"]?.ToString(), CardCount = Convert.ToInt32(setJson.GetValueOrDefault("card_count") ?? 0), SetType = setJson["set_type"]?.ToString(), SetIcon = setJson["icon_svg_uri"]?.ToString(), ReleaseDate = (DateTime)(DateTime.TryParse(setJson["released_at"]?.ToString(), out var dt) ? dt : (DateTime?)null) }; _mtgContext.Sets.Add(set); } counter++; if (counter >= 500) { counter = 0; await _mtgContext.SaveChangesAsync(); } } } } await _mtgContext.SaveChangesAsync(); } private async Task ProcessCardsAsync() { // --- Caches --- Dictionary superTypeCache = _mtgContext.Supertypes.ToDictionary(s => s.TypeEn); Dictionary typeCache = _mtgContext.Types.ToDictionary(t => t.TypeEn); Dictionary setCache = _mtgContext.Sets.ToDictionary(s => s.SetCode); using var fileStream = File.OpenRead(CardFile); using var reader = new StreamReader(fileStream); using var json = new JsonTextReader(reader); var serializer = new JsonSerializer(); int counter = 0; while (await json.ReadAsync()) { if (json.TokenType != JsonToken.StartObject) continue; var cardJson = serializer.Deserialize>(json); if (cardJson == null) continue; try { if (cardJson.Keys.Contains("card_faces")) { if (cardJson.TryGetValue("card_faces", out var cardFaces) && cardFaces is JArray dataArray) { string cardId = cardJson["id"].ToString(); int faceCounter = 0; foreach (JObject face in dataArray) { cardJson["id"] = cardId + faceCounter; cardJson["type_line"] = face["type_line"]; cardJson["oracle_text"] = face["oracle_text"]; if (face.TryGetValue("flavor_text", out var flavText)) cardJson["flavor_text"] = flavText; cardJson["name"] = face["name"]; cardJson["mana_cost"] = face["mana_cost"]; if(faceCounter == dataArray.Count - 1) { cardJson["otherFace"] = cardId + 0; } else { cardJson["otherFace"] = cardId + faceCounter + 1; } string cardLang = cardJson["lang"]?.ToString(); if (cardLang == null) return; if (cardLang != "en" && cardLang != "de") continue; if (cardLang == "de") { cardJson["printed_type_line"] = face["printed_type_line"]; cardJson["printed_text"] = face["printed_text"]; cardJson["printed_name"] = face["printed_name"]; } await addCard(cardJson, superTypeCache, typeCache, setCache); faceCounter++; } } } else { await addCard(cardJson,superTypeCache,typeCache,setCache); } counter++; if (counter >= 500) { counter = 0; await _mtgContext.SaveChangesAsync(); if (!_ftsCreated) { _mtgContext.EnsureFtsInitialized(); _ftsCreated = true; } //_mtgContext.ChangeTracker.Clear(); } } catch(Exception ex) { throw new Exception("Error At Card: " + JsonConvert.SerializeObject(cardJson,Formatting.Indented) ,ex); } } await _mtgContext.SaveChangesAsync(); _mtgContext.RebuildCardPrintingsFts(); } private async Task addCard(Dictionary cardJson, Dictionary superTypeCache, Dictionary typeCache, Dictionary setCache) { string cardLang = cardJson["lang"]?.ToString(); if (cardLang == null) return; if (cardLang != "en" && cardLang != "de") return; string typeline = cardJson["type_line"]?.ToString() ?? ""; string printTypeline = cardLang == "de" ? cardJson.TryGetValue("printed_type_line", out object? value) ? value?.ToString() : cardJson["type_line"].ToString() : null; string cardName = cardJson["name"]?.ToString() ?? ""; string setCode = cardJson["set"]?.ToString() ?? ""; string collectorNumber = cardJson["collector_number"]?.ToString(); if (collectorNumber == null) return; if (!setCache.TryGetValue(setCode, out var set)) return; // --- Supertypen --- string sSuperType = typeline.Split('—')[0].Trim(); string? printedSuperType = cardLang == "de" && printTypeline != null ? printTypeline.Split('—')[0].Trim() : ""; if (!superTypeCache.TryGetValue(sSuperType, out var superType)) { superType = new Supertype { TypeEn = sSuperType, TypeDe = printedSuperType }; _mtgContext.Supertypes.Add(superType); superTypeCache[sSuperType] = superType; } else { if (superType.TypeDe == null || superType.TypeDe == string.Empty) superType.TypeDe = printedSuperType; } // --- Typen --- var types = typeline.Split('—'); if (types.Length > 1) { types = types[1].Trim().Split(' ', StringSplitOptions.RemoveEmptyEntries); } var printedTypes = cardLang == "de" && printTypeline != null ? printTypeline.Split('—') : null; if (printedTypes != null && printedTypes.Length > 1) printedTypes = printedTypes[1].Trim().Split(' '); var mtgTypes = new List(); for (int i = 0; i < types.Length; i++) { string typeName = types[i]; if (!typeCache.TryGetValue(typeName, out var mtgType)) { mtgType = new MTGType { TypeEn = typeName, TypeDe = cardLang == "de" && printedTypes != null && i < printedTypes.Length ? printedTypes[i] : "" }; _mtgContext.Types.Add(mtgType); typeCache[typeName] = mtgType; } else { if (mtgType.TypeDe == null || mtgType.TypeDe == string.Empty) mtgType.TypeDe = cardLang == "de" && printedTypes != null && i < printedTypes.Length ? printedTypes[i] : ""; } mtgTypes.Add(mtgType); } // --- Karte identifizieren --- var card = await _mtgContext.Cards .Include(c => c.Printings) .FirstOrDefaultAsync(c => c.CollectorNumber == collectorNumber && c.SetCode == setCode); var cardPrinting = new CardPrinting { CardId = cardJson["id"]?.ToString(), Lang = cardLang, Name = cardLang == "de" && cardJson.ContainsKey("printed_name") && cardJson["printed_name"] != null ? cardJson["printed_name"]?.ToString() : cardJson["name"]?.ToString(), OracleText = cardLang == "de" && cardJson.ContainsKey("printed_text") ? cardJson.GetValueOrDefault("printed_text")?.ToString() : cardJson.GetValueOrDefault("oracle_text")?.ToString(), FlavorText = cardJson.GetValueOrDefault("flavor_text")?.ToString(), ImageUriSmall = (cardJson.GetValueOrDefault("image_uris") as JObject)?["small"]?.ToString(), ImageUriNormal = (cardJson.GetValueOrDefault("image_uris") as JObject)?["normal"]?.ToString(), ImageUriLarge = (cardJson.GetValueOrDefault("image_uris") as JObject)?["large"]?.ToString() }; if (card == null) { // Neue Karte anlegen card = new Card { Id = cardJson["id"]?.ToString(), //CardName = cardName, //SetCode = setCode, SuperTypeId = superType.TypeId, CollectorNumber = collectorNumber, ManaCost = cardJson.GetValueOrDefault("mana_cost")?.ToString(), Foil = Convert.ToBoolean(cardJson.GetValueOrDefault("foil") ?? false), NonFoil = Convert.ToBoolean(cardJson.GetValueOrDefault("nonfoil") ?? false), Rarity = cardJson["rarity"]?.ToString(), Power = cardJson.GetValueOrDefault("power")?.ToString(), Toughness = cardJson.GetValueOrDefault("toughness")?.ToString(), Cmc = Convert.ToInt32(cardJson.GetValueOrDefault("cmc") ?? 0), OtherFace = cardJson.GetValueOrDefault("otherFace")?.ToString(), SuperType = superType, Set = set, Printings = new List { cardPrinting } }; _mtgContext.Cards.Add(card); foreach (var mtgType in mtgTypes) _mtgContext.CardTypes.Add(new CardType { Card = card, Type = mtgType }); } else { // Prüfen, ob Printing für diese Sprache existiert var existingPrinting = card.Printings.FirstOrDefault(p => p.Lang == cardLang); if (existingPrinting == null) { card.Printings.Add(cardPrinting); } else { // Update vorhandenes Printing, falls sich etwas geändert hat bool changed = false; if (existingPrinting.Name != cardPrinting.Name) { existingPrinting.Name = cardPrinting.Name; changed = true; } if (existingPrinting.OracleText != cardPrinting.OracleText) { existingPrinting.OracleText = cardPrinting.OracleText; changed = true; } if (existingPrinting.FlavorText != cardPrinting.FlavorText) { existingPrinting.FlavorText = cardPrinting.FlavorText; changed = true; } if (existingPrinting.ImageUriSmall != cardPrinting.ImageUriSmall) { existingPrinting.ImageUriSmall = cardPrinting.ImageUriSmall; changed = true; } if (existingPrinting.ImageUriNormal != cardPrinting.ImageUriNormal) { existingPrinting.ImageUriNormal = cardPrinting.ImageUriNormal; changed = true; } if (existingPrinting.ImageUriLarge != cardPrinting.ImageUriLarge) { existingPrinting.ImageUriLarge = cardPrinting.ImageUriLarge; changed = true; } if (changed) { //_mtgContext.CardPrintings.Update(existingPrinting); } } } } #endregion } }