enhanced domain extraction and bulk import functionality

pages/api/admin/bulk-import.ts

@@ -0,0 +1,80 @@
+import type { NextApiRequest, NextApiResponse } from "next"
+import sqlite3 from "sqlite3"
+import path from "path"
+
+interface BulkImportItem {
+  domain: string
+  risk_level: number
+  categories: string[]
+  description?: string
+}
+
+export default async function handler(req: NextApiRequest, res: NextApiResponse) {
+  if (req.method !== "POST") return res.status(405).json({ error: "Method not allowed" })
+  
+  const { sources } = req.body as { sources: BulkImportItem[] }
+  
+  if (!sources || !Array.isArray(sources)) {
+    return res.status(400).json({ error: "Sources array required" })
+  }
+  
+  const dbPath = path.join(process.cwd(), "database", "antihoax.db")
+  const db = new sqlite3.Database(dbPath)
+  
+  try {
+    let imported = 0
+    let skipped = 0
+    
+    for (const source of sources) {
+      if (!source.domain || !source.risk_level) {
+        skipped++
+        continue
+      }
+      
+      // Check if domain already exists
+      const existing = await new Promise<any>((resolve, reject) => {
+        db.get(
+          "SELECT id FROM sources WHERE domain = ?",
+          [source.domain],
+          (err, row) => {
+            if (err) reject(err)
+            else resolve(row)
+          }
+        )
+      })
+      
+      if (existing) {
+        skipped++
+        continue
+      }
+      
+      // Insert new source
+      await new Promise<void>((resolve, reject) => {
+        db.run(
+          `INSERT INTO sources (domain, title, risk_level, status, description, created_at)
+           VALUES (?, ?, ?, 'verified', ?, datetime('now'))`,
+          [source.domain, source.domain, source.risk_level, source.description || ''],
+          function(err) {
+            if (err) reject(err)
+            else resolve()
+          }
+        )
+      })
+      
+      imported++
+    }
+    
+    res.json({ 
+      success: true, 
+      imported,
+      skipped,
+      total: sources.length
+    })
+    
+  } catch (error) {
+    console.error('Bulk import error:', error)
+    res.status(500).json({ error: "Import failed" })
+  } finally {
+    db.close()
+  }
+}

pages/api/sources/check.ts

@@ -25,6 +25,20 @@ function extractDomain(url: string): string {
     domain = domain.replace(/^www\./, '')
     domain = domain.replace(/^m\./, '')
     domain = domain.replace(/^mobile\./, '')
+    domain = domain.replace(/^amp\./, '')
+    
+    // Handle subdomains for known patterns - extract main domain for common TLDs
+    if (domain.includes('.')) {
+      const parts = domain.split('.')
+      if (parts.length > 2) {
+        const tld = parts[parts.length - 1]
+        const sld = parts[parts.length - 2]
+        // Extract main domain for common TLDs
+        if (['com', 'org', 'net', 'sk', 'cz', 'hu', 'pl', 'eu', 'info'].includes(tld)) {
+          domain = `${sld}.${tld}`
+        }
+      }
+    }
     
     return domain
   } catch {