feat: Enhance CSV parser to dynamically map quantity, name, finish, and ID columns from headers for robust custom imports.
All checks were successful
Build and Deploy / build (push) Successful in 1m20s

This commit is contained in:
2025-12-16 15:23:59 +01:00
parent dcbc484a1c
commit 8a40bc6ca4
3 changed files with 107 additions and 35 deletions

View File

@@ -8,3 +8,4 @@
## Recent Completions ## Recent Completions
- [Game Battlefield & Manual Mode](./devlog/2025-12-14-234500_game_battlefield_plan.md): Completed. - [Game Battlefield & Manual Mode](./devlog/2025-12-14-234500_game_battlefield_plan.md): Completed.
- [Helm Chart Config](./devlog/2025-12-14-214500_helm_config.md): Completed. - [Helm Chart Config](./devlog/2025-12-14-214500_helm_config.md): Completed.
- [CSV Import Robustness](./devlog/2025-12-16-152253_csv_import_robustness.md): Completed. Enhanced CSV parser to dynamically map columns from headers, supporting custom user imports.

View File

@@ -0,0 +1,21 @@
# CSV Import Robustness Update
## Background
The user provided a specific CSV format associated with typical automated imports. The requirement was to extract relevant information (Quantity, Name, Finish, Scryfall ID) while ignoring other fields (such as Condition, Date Added, etc.).
## Changes
- Refactored `src/client/src/services/CardParserService.ts` to implement dynamic header parsing.
- The `parse` method now:
- Detects if the first line is a CSV header containing "Quantity" and "Name".
- Maps columns to indices based on the header.
- Specifically looks for `Quantity`, `Name`, `Finish`, and `Scryfall ID` (checking common variations like 'scryfall_id', 'id', 'uuid').
- Uses strictly mapped columns if a header is detected, ensuring other fields are ignored as requested.
- Falls back gracefully to previous generic parsing logic if no matching header is found, preserving backward compatibility with Arena/MTGO exports and simple lists.
## Verification
- Verified manually via a test script that the provided CSV content parses correctly into the `CardIdentifier` memory structure.
- The extraction correctly identifies Quantity, Name, Finish (Normal/Foil), and Scryfall UUID.
## Next Steps
- Ensure the frontend `CubeManager` works seamlessly with this update (no changes needed there as it uses the service).

View File

@@ -11,53 +11,105 @@ export class CardParserService {
const rawCardList: CardIdentifier[] = []; const rawCardList: CardIdentifier[] = [];
const uuidRegex = /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/i; const uuidRegex = /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/i;
let colMap = { qty: 0, name: 1, finish: 2, id: -1, found: false };
// Check header to determine column indices dynamically
if (lines.length > 0) {
const headerLine = lines[0].toLowerCase();
// Heuristic: if it has Quantity and Name, it's likely our CSV
if (headerLine.includes('quantity') && headerLine.includes('name')) {
const headers = this.parseCsvLine(lines[0]).map(h => h.toLowerCase().trim());
const qtyIndex = headers.indexOf('quantity');
const nameIndex = headers.indexOf('name');
if (qtyIndex !== -1 && nameIndex !== -1) {
colMap.qty = qtyIndex;
colMap.name = nameIndex;
colMap.finish = headers.indexOf('finish');
// Find ID column: could be 'scryfall id', 'scryfall_id', 'id'
colMap.id = headers.findIndex(h => h === 'scryfall id' || h === 'scryfall_id' || h === 'id' || h === 'uuid');
colMap.found = true;
// Remove header row
lines.shift();
}
}
}
lines.forEach(line => { lines.forEach(line => {
// Skip header // Skip generic header repetition if it occurs
if (line.toLowerCase().startsWith('quantity') && line.toLowerCase().includes('name')) return; if (line.toLowerCase().startsWith('quantity') && line.toLowerCase().includes('name')) return;
// Try parsing as CSV line first if we detected a header or if it looks like CSV
const parts = this.parseCsvLine(line);
// If we have a detected map, use it strict(er)
if (colMap.found && parts.length > Math.max(colMap.qty, colMap.name)) {
const qty = parseInt(parts[colMap.qty]);
if (!isNaN(qty)) {
const name = parts[colMap.name];
let finish: 'foil' | 'normal' | undefined = undefined;
if (colMap.finish !== -1 && parts[colMap.finish]) {
const finishRaw = parts[colMap.finish].toLowerCase();
finish = (finishRaw === 'foil' || finishRaw === 'etched') ? 'foil' : (finishRaw === 'normal' ? 'normal' : undefined);
} else if (!colMap.found) {
// Legacy fallback for default indices if header wasn't found but we are in this block (shouldn't happen with colMap.found=true logic)
const finishRaw = parts[2]?.toLowerCase();
finish = (finishRaw === 'foil' || finishRaw === 'etched') ? 'foil' : (finishRaw === 'normal' ? 'normal' : undefined);
}
let idValue: string | null = null;
// If we have an ID column, look there
if (colMap.id !== -1 && parts[colMap.id]) {
const match = parts[colMap.id].match(uuidRegex);
if (match) idValue = match[0];
}
// If not found in column (or no column), check if there's a UUID anywhere in the line?
// The user said "ignore other fields". So strictly adhering to columns is better.
// BUT, to be safe for mixed usages (e.g. if ID is missing in col but present elsewhere? Unlikely).
// Let's stick to the mapped column if available.
// If we didn't find an ID in the specific column, but we have a generic UUID in the line?
// The original logic did `parts.find`.
// If `colMap.found` is true, we should trust it.
if (idValue) {
rawCardList.push({ type: 'id', value: idValue, quantity: qty, finish });
return;
} else if (name) {
rawCardList.push({ type: 'name', value: name, quantity: qty, finish });
return;
}
}
}
// --- Fallback / Original Logic for non-header formats or failed parsings ---
const idMatch = line.match(uuidRegex); const idMatch = line.match(uuidRegex);
if (idMatch) { if (idMatch) {
// Extract quantity if present before ID, otherwise default to 1 // It has a UUID, try to extract generic CSV info if possible
// Simple check: Look for "Nx ID" or "N, ID" pattern?
// The previous/standard logic usually treats ID lines as 1x unless specified.
// Let's try to find a quantity at the start if it exists differently from UUID.
// But usually UUID lines are direct from export.
// But our CSV template puts ID at the end.
// If UUID is present anywhere in the line, we might trust it over the name.
// Let's stick to the previous logic: if UUID is found, use it.
// BUT, we should try to parse the whole CSV line if possible to get Finish and Quantity.
// Let's parse with CSV logic first.
const parts = this.parseCsvLine(line);
if (parts.length >= 2) { if (parts.length >= 2) {
const qty = parseInt(parts[0]); const qty = parseInt(parts[0]);
// If valid CSV structure
if (!isNaN(qty)) { if (!isNaN(qty)) {
// const name = parts[1]; // We can keep name for reference, but we use ID if present // Assuming default 0=Qty, 2=Finish if no header map found
const finishRaw = parts[2]?.toLowerCase(); const finishRaw = parts[2]?.toLowerCase();
const finish = (finishRaw === 'foil' || finishRaw === 'etched') ? 'foil' : (finishRaw === 'normal' ? 'normal' : undefined); const finish = (finishRaw === 'foil' || finishRaw === 'etched') ? 'foil' : (finishRaw === 'normal' ? 'normal' : undefined);
// If the last part has UUID, use it. // Use the regex match found
const uuidPart = parts.find(p => uuidRegex.test(p)); rawCardList.push({ type: 'id', value: idMatch[0], quantity: qty, finish });
if (uuidPart) { return;
const uuid = uuidPart.match(uuidRegex)![0];
rawCardList.push({ type: 'id', value: uuid, quantity: qty, finish });
return;
}
} }
} }
// Just ID flow
// Fallback ID logic rawCardList.push({ type: 'id', value: idMatch[0], quantity: 1 });
rawCardList.push({ type: 'id', value: idMatch[0], quantity: 1 }); // Default simple UUID match
return; return;
} }
// Not an ID match, try parsing as name // Name-based generic parsing (Arena/MTGO or simple CSV without ID)
const parts = this.parseCsvLine(line);
if (parts.length >= 2 && !isNaN(parseInt(parts[0]))) { if (parts.length >= 2 && !isNaN(parseInt(parts[0]))) {
// It looks like result of our CSV: Quantity, Name, Finish, ...
const quantity = parseInt(parts[0]); const quantity = parseInt(parts[0]);
const name = parts[1]; const name = parts[1];
const finishRaw = parts[2]?.toLowerCase(); const finishRaw = parts[2]?.toLowerCase();
@@ -69,18 +121,16 @@ export class CardParserService {
} }
} }
// Fallback to simple Arena/MTGO text format: "4 Lightning Bolt" // "4 Lightning Bolt" format
const cleanLine = line.replace(/['"]/g, ''); const cleanLine = line.replace(/['"]/g, '');
const simpleMatch = cleanLine.match(/^(\d+)[xX\s]+(.+)$/); const simpleMatch = cleanLine.match(/^(\d+)[xX\s]+(.+)$/);
if (simpleMatch) { if (simpleMatch) {
let name = simpleMatch[2].trim(); let name = simpleMatch[2].trim();
// cleanup name = name.replace(/\s*[\(\[].*?[\)\]]/g, '');
name = name.replace(/\s*[\(\[].*?[\)\]]/g, ''); // remove set codes name = name.replace(/\s+\d+$/, '');
name = name.replace(/\s+\d+$/, ''); // remove collector number
rawCardList.push({ type: 'name', value: name, quantity: parseInt(simpleMatch[1]) }); rawCardList.push({ type: 'name', value: name, quantity: parseInt(simpleMatch[1]) });
} else { } else {
// Maybe just "Lightning Bolt" (1x)
let name = cleanLine.trim(); let name = cleanLine.trim();
if (name) { if (name) {
rawCardList.push({ type: 'name', value: name, quantity: 1 }); rawCardList.push({ type: 'name', value: name, quantity: 1 });