import { useMemo, useState } from "react"; import { Controller, useForm } from "react-hook-form"; import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; import { format } from "date-fns"; import { CalendarIcon, ArrowDown, ArrowUp, Sparkles } from "lucide-react"; import { ApiError, api } from "@/lib/api"; import { cn } from "@/lib/utils"; import type { JsonObject } from "@/components/json-fields"; import { Button } from "@/components/ui/button"; import { Card, CardContent, CardDescription, CardHeader, CardTitle, } from "@/components/ui/card"; import { Checkbox } from "@/components/ui/checkbox"; import { Input } from "@/components/ui/input"; import { Textarea } from "@/components/ui/textarea"; import { Calendar } from "@/components/ui/calendar"; import { Popover, PopoverContent, PopoverTrigger, } from "@/components/ui/popover"; import { SchemaAwareJsonField } from "@/components/schema-builder"; import { Dialog, DialogClose, DialogContent, DialogDescription, DialogFooter, DialogHeader, DialogTitle, } from "@/components/ui/dialog"; import { getSchema } from "@/schemas/loader"; import { buildDefaultValue } from "@/schemas/utils"; export interface Scraper { id: string; params: string | null; frequency: string | null; task_name: string | null; property_types: string | null; last_seen_days: number | null; first_seen_days: number | null; page_size: number | null; max_pages: number | null; enabled: number | null; enrich_llm: number | null; only_match: number | null; once: number | null; } interface ScraperFormValues { params: JsonObject; frequency: string; task_name: string; property_types: string; last_seen_days: string; first_seen_days: string; page_size: string; max_pages: string; enabled: boolean; enrich_llm: boolean; only_match: boolean; once: boolean; } interface CreateScraperPayload { id?: string; params?: string | null; frequency?: string | null; task_name?: string | null; property_types?: string | null; last_seen_days?: number | null; first_seen_days?: number | null; page_size?: number | null; max_pages?: number | null; enabled?: number | null; enrich_llm?: number | null; only_match?: number | null; once?: number | null; } const scraperSchema = getSchema("scraper"); function getErrorMessage(error: unknown) { if (error instanceof ApiError) { if (typeof error.payload === "string" && error.payload.trim() !== "") { return error.payload; } if (error.payload && typeof error.payload === "object") { try { return JSON.stringify(error.payload); } catch (stringifyError) { return (stringifyError as Error).message; } } return `Requete rejetee (${error.status}).`; } if (error instanceof Error) { return error.message; } return "Erreur inconnue."; } function parseInteger(value: string) { if (value.trim() === "") { return undefined; } const parsed = Number.parseInt(value, 10); if (Number.isNaN(parsed)) { return undefined; } return parsed; } function transformPayload(values: ScraperFormValues): CreateScraperPayload { const payload: CreateScraperPayload = {}; const paramsObject = values.params ?? {}; console.log("transformPayload - paramsObject:", paramsObject); console.log("transformPayload - paramsObject keys:", Object.keys(paramsObject)); if (Object.keys(paramsObject).length > 0) { payload.params = JSON.stringify(paramsObject); } const frequency = values.frequency.trim(); if (frequency) { payload.frequency = frequency; } const taskName = values.task_name.trim(); if (taskName) { payload.task_name = taskName; } const propertyTypes = values.property_types.trim(); if (propertyTypes) { payload.property_types = propertyTypes; } const lastSeen = parseInteger(values.last_seen_days); if (lastSeen !== undefined) { payload.last_seen_days = lastSeen; } const firstSeen = parseInteger(values.first_seen_days); if (firstSeen !== undefined) { payload.first_seen_days = firstSeen; } const pageSize = parseInteger(values.page_size); if (pageSize !== undefined) { payload.page_size = pageSize; } const maxPages = parseInteger(values.max_pages); if (maxPages !== undefined) { payload.max_pages = maxPages; } payload.enabled = values.enabled ? 1 : 0; payload.enrich_llm = values.enrich_llm ? 1 : 0; payload.only_match = values.only_match ? 1 : 0; // Ne pas envoyer 'once' si non coché if (values.once) { payload.once = 1; } return payload; } function createDefaultScraperValues(): ScraperFormValues { return { params: buildDefaultValue(scraperSchema) as JsonObject, frequency: "", task_name: "", property_types: "", last_seen_days: "", first_seen_days: "", page_size: "", max_pages: "", enabled: true, enrich_llm: false, only_match: false, once: false, }; } function formatParamsDisplay(params: string | null) { if (!params) { return null; } try { return JSON.stringify(JSON.parse(params), null, 2); } catch (parseError) { return params; } } export function ScrapersTab() { const [paramsValid, setParamsValid] = useState(true); const [editingScraperId, setEditingScraperId] = useState(null); const [statusMessage, setStatusMessage] = useState(null); const [isCountDialogOpen, setIsCountDialogOpen] = useState(false); const [countFirstSeenDate, setCountFirstSeenDate] = useState(undefined); const [countLastSeenDate, setCountLastSeenDate] = useState(undefined); const [isAiDialogOpen, setIsAiDialogOpen] = useState(false); const [aiPrompt, setAiPrompt] = useState(""); const queryClient = useQueryClient(); const initialValues = useMemo(() => createDefaultScraperValues(), []); const form = useForm({ defaultValues: initialValues, mode: "onBlur", }); const scrapersQuery = useQuery({ queryKey: ["scrapers"], queryFn: () => api.get("/scrapers"), }); const createScraperMutation = useMutation({ mutationFn: (values: ScraperFormValues) => api.post( "/scrapers", transformPayload(values), ), onSuccess: () => { queryClient.invalidateQueries({ queryKey: ["scrapers"] }); setParamsValid(true); setEditingScraperId(null); form.reset(createDefaultScraperValues()); setStatusMessage("Scraper cree avec succes."); }, }); const updateScraperMutation = useMutation({ mutationFn: ({ id, data }: { id: string; data: ScraperFormValues }) => { const payload = transformPayload(data); return api.put(`/scrapers/${id}`, payload); }, onSuccess: () => { queryClient.invalidateQueries({ queryKey: ["scrapers"] }); setEditingScraperId(null); setParamsValid(true); form.reset(createDefaultScraperValues()); setStatusMessage("Scraper mis a jour."); }, }); const deleteScraperMutation = useMutation({ mutationFn: (id: string) => api.delete(`/scrapers/${id}`), onSuccess: (_, id) => { if (editingScraperId === id) { setEditingScraperId(null); setParamsValid(true); form.reset(createDefaultScraperValues()); } queryClient.invalidateQueries({ queryKey: ["scrapers"] }); setStatusMessage("Scraper supprime."); }, }); type ScraperCountResponse = { count: number }; const countScraperMutation = useMutation({ mutationFn: (payload: { values: ScraperFormValues; firstSeenDate?: Date; lastSeenDate?: Date }) => { const basePayload = transformPayload(payload.values); // Override first_seen_days and last_seen_days with date-based calculations if dates are provided if (payload.firstSeenDate) { const daysSinceFirstSeen = Math.floor((new Date().getTime() - payload.firstSeenDate.getTime()) / (1000 * 60 * 60 * 24)); basePayload.first_seen_days = daysSinceFirstSeen; } if (payload.lastSeenDate) { const daysSinceLastSeen = Math.floor((new Date().getTime() - payload.lastSeenDate.getTime()) / (1000 * 60 * 60 * 24)); basePayload.last_seen_days = daysSinceLastSeen; } console.log("Count payload being sent:", basePayload); return api.post( "/scrapers/count", basePayload, ); }, }); type AiGenerateResponse = { params: JsonObject }; const aiGenerateMutation = useMutation({ mutationFn: (prompt: string) => api.post<{ prompt: string }, AiGenerateResponse>("/ai/generate-scraper", { prompt }), onSuccess: (data) => { if (data.params) { form.setValue("params", data.params); setStatusMessage("Scraper genere par IA avec succes."); setIsAiDialogOpen(false); setAiPrompt(""); } }, }); const handleAiGenerate = () => { if (aiPrompt.trim()) { aiGenerateMutation.mutate(aiPrompt.trim()); } }; const onSubmit = form.handleSubmit(async (values) => { setStatusMessage(null); if (editingScraperId) { await updateScraperMutation.mutateAsync({ id: editingScraperId, data: values, }); } else { await createScraperMutation.mutateAsync(values); } }); const formError = (createScraperMutation.isError && getErrorMessage(createScraperMutation.error)) || (updateScraperMutation.isError && getErrorMessage(updateScraperMutation.error)) || null; const handleOpenCountDialog = async () => { setStatusMessage(null); const isFormValid = await form.trigger(); if (!isFormValid || !paramsValid) { return; } countScraperMutation.reset(); setCountFirstSeenDate(undefined); setCountLastSeenDate(undefined); setIsCountDialogOpen(true); }; const handleCountWithDates = () => { countScraperMutation.mutate({ values: form.getValues(), firstSeenDate: countFirstSeenDate, lastSeenDate: countLastSeenDate, }); }; const scrapers = useMemo( () => scrapersQuery.data ?? [], [scrapersQuery.data], ); const scrollToTop = () => { window.scrollTo({ top: 0, behavior: "instant" }); }; const scrollToList = () => { const listElement = document.getElementById("scrapers-list"); if (listElement) { listElement.scrollIntoView({ behavior: "instant" }); } }; return (
Creer un scraper Definissez la configuration et les parametres d'execution du scraper.

Identifiant unique pour reperer ce scraper

Expression cron pour planifier l'execution

Types de proprietes a rechercher (separes par virgule)

{ if (!value || value.trim() === "") { return true; // Champ optionnel } const validTypes = ["immeuble", "appartement", "maison"]; const types = value .split(",") .map((t) => t.trim().toLowerCase()) .filter((t) => t !== ""); const invalidTypes = types.filter( (t) => !validTypes.includes(t) ); if (invalidTypes.length > 0) { return `Types invalides: ${invalidTypes.join(", ")}. Valeurs autorisées: ${validTypes.join(", ")}`; } return true; }, })} /> {form.formState.errors.property_types && (

{form.formState.errors.property_types.message}

)}

Annonces vues il y a maximum N jours

Annonces apparues il y a minimum N jours

Nombre de resultats par page

Nombre maximum de pages a parcourir

(

Active ou desactive l'execution automatique

)} /> (

Analyse les annonces avec un modele de langage

)} /> (

Ne retourne que les correspondances exactes

)} /> (

Execute le scraper une seule fois

)} />
( field.onChange(next)} label="Parametres" description="Structure schemaisee conformement au schema scraper." onValidationChange={(payload) => setParamsValid(payload.valid) } /> )} />
{statusMessage ? (

{statusMessage}

) : null} {formError ? (

{formError}

) : null} {!paramsValid ? (

Corrigez les erreurs du schema avant l'envoi.

) : null} {editingScraperId ? ( ) : null}
Scrapers existants {scrapersQuery.isLoading ? "Chargement des scrapers..." : scrapers.length === 0 ? "Aucun scraper enregistre." : "Liste recue depuis l'API."}
{scrapers.length > 0 && ( )}
{scrapersQuery.isError ? (

{getErrorMessage(scrapersQuery.error)}

) : ( {scrapers.map((scraper) => { const paramsDisplay = formatParamsDisplay(scraper.params); return ( ); })}
Nom Frequence Parametres Etat Limites Actions
{scraper.task_name ?? "-"} {scraper.frequency ?? "-"}
{paramsDisplay ? (
                              {paramsDisplay}
                            
) : ( - )} {scraper.params && ( )}
Actif: {scraper.enabled ? "Oui" : "Non"} LLM: {scraper.enrich_llm ? "Oui" : "Non"} Strict: {scraper.only_match ? "Oui" : "Non"} Une fois: {scraper.once ? "Oui" : "Non"}
Derniere vue: {scraper.last_seen_days ?? "-"} Premiere vue: {scraper.first_seen_days ?? "-"} Page size: {scraper.page_size ?? "-"} Max pages: {scraper.max_pages ?? "-"}
)}
{ setIsCountDialogOpen(open); if (!open) { countScraperMutation.reset(); setCountFirstSeenDate(undefined); setCountLastSeenDate(undefined); } }} > Estimation du nombre d'annonces Calcul base sur la configuration actuelle du scraper. Vous pouvez filtrer par période.

Annonces vues pour la première fois après cette date

Annonces vues pour la dernière fois avant cette date

{countScraperMutation.isPending ? (

Calcul en cours...

) : countScraperMutation.isError ? (

{getErrorMessage(countScraperMutation.error)}

) : countScraperMutation.data ? (

Ce scraper correspond a {" "} {countScraperMutation.data.count}{" "} annonce(s) potentielle(s).

) : (

Cliquez sur "Calculer" pour obtenir le nombre d'annonces.

)}
{ setIsAiDialogOpen(open); if (!open) { aiGenerateMutation.reset(); setAiPrompt(""); } }} > Generer un scraper avec l'IA Decrivez en langage naturel le type de scraper que vous souhaitez creer. L'IA generera les parametres JSON correspondants.