1106 lines
38 KiB
TypeScript
1106 lines
38 KiB
TypeScript
import { useMemo, useState } from "react";
|
|
import { Controller, useForm } from "react-hook-form";
|
|
import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
|
|
import { format } from "date-fns";
|
|
import { CalendarIcon, ArrowDown, ArrowUp, Sparkles } from "lucide-react";
|
|
|
|
import { ApiError, api } from "@/lib/api";
|
|
import { cn } from "@/lib/utils";
|
|
import type { JsonObject } from "@/components/json-fields";
|
|
import { Button } from "@/components/ui/button";
|
|
import {
|
|
Card,
|
|
CardContent,
|
|
CardDescription,
|
|
CardHeader,
|
|
CardTitle,
|
|
} from "@/components/ui/card";
|
|
import { Checkbox } from "@/components/ui/checkbox";
|
|
import { Input } from "@/components/ui/input";
|
|
import { Textarea } from "@/components/ui/textarea";
|
|
import { Calendar } from "@/components/ui/calendar";
|
|
import {
|
|
Popover,
|
|
PopoverContent,
|
|
PopoverTrigger,
|
|
} from "@/components/ui/popover";
|
|
import { SchemaAwareJsonField } from "@/components/schema-builder";
|
|
import {
|
|
Dialog,
|
|
DialogClose,
|
|
DialogContent,
|
|
DialogDescription,
|
|
DialogFooter,
|
|
DialogHeader,
|
|
DialogTitle,
|
|
} from "@/components/ui/dialog";
|
|
import { getSchema } from "@/schemas/loader";
|
|
import { buildDefaultValue } from "@/schemas/utils";
|
|
|
|
export interface Scraper {
|
|
id: string;
|
|
params: string | null;
|
|
frequency: string | null;
|
|
task_name: string | null;
|
|
property_types: string | null;
|
|
last_seen_days: number | null;
|
|
first_seen_days: number | null;
|
|
page_size: number | null;
|
|
max_pages: number | null;
|
|
enabled: number | null;
|
|
enrich_llm: number | null;
|
|
only_match: number | null;
|
|
once: number | null;
|
|
}
|
|
|
|
interface ScraperFormValues {
|
|
params: JsonObject;
|
|
frequency: string;
|
|
task_name: string;
|
|
property_types: string;
|
|
last_seen_days: string;
|
|
first_seen_days: string;
|
|
page_size: string;
|
|
max_pages: string;
|
|
enabled: boolean;
|
|
enrich_llm: boolean;
|
|
only_match: boolean;
|
|
once: boolean;
|
|
}
|
|
|
|
interface CreateScraperPayload {
|
|
id?: string;
|
|
params?: string | null;
|
|
frequency?: string | null;
|
|
task_name?: string | null;
|
|
property_types?: string | null;
|
|
last_seen_days?: number | null;
|
|
first_seen_days?: number | null;
|
|
page_size?: number | null;
|
|
max_pages?: number | null;
|
|
enabled?: number | null;
|
|
enrich_llm?: number | null;
|
|
only_match?: number | null;
|
|
once?: number | null;
|
|
}
|
|
|
|
const scraperSchema = getSchema("scraper");
|
|
|
|
function getErrorMessage(error: unknown) {
|
|
if (error instanceof ApiError) {
|
|
if (typeof error.payload === "string" && error.payload.trim() !== "") {
|
|
return error.payload;
|
|
}
|
|
|
|
if (error.payload && typeof error.payload === "object") {
|
|
try {
|
|
return JSON.stringify(error.payload);
|
|
} catch (stringifyError) {
|
|
return (stringifyError as Error).message;
|
|
}
|
|
}
|
|
|
|
return `Requete rejetee (${error.status}).`;
|
|
}
|
|
|
|
if (error instanceof Error) {
|
|
return error.message;
|
|
}
|
|
|
|
return "Erreur inconnue.";
|
|
}
|
|
|
|
function parseInteger(value: string) {
|
|
if (value.trim() === "") {
|
|
return undefined;
|
|
}
|
|
|
|
const parsed = Number.parseInt(value, 10);
|
|
|
|
if (Number.isNaN(parsed)) {
|
|
return undefined;
|
|
}
|
|
|
|
return parsed;
|
|
}
|
|
|
|
function transformPayload(values: ScraperFormValues): CreateScraperPayload {
|
|
const payload: CreateScraperPayload = {};
|
|
|
|
const paramsObject = values.params ?? {};
|
|
console.log("transformPayload - paramsObject:", paramsObject);
|
|
console.log("transformPayload - paramsObject keys:", Object.keys(paramsObject));
|
|
|
|
if (Object.keys(paramsObject).length > 0) {
|
|
payload.params = JSON.stringify(paramsObject);
|
|
}
|
|
|
|
const frequency = values.frequency.trim();
|
|
if (frequency) {
|
|
payload.frequency = frequency;
|
|
}
|
|
|
|
const taskName = values.task_name.trim();
|
|
if (taskName) {
|
|
payload.task_name = taskName;
|
|
}
|
|
|
|
const propertyTypes = values.property_types.trim();
|
|
if (propertyTypes) {
|
|
payload.property_types = propertyTypes;
|
|
}
|
|
|
|
const lastSeen = parseInteger(values.last_seen_days);
|
|
if (lastSeen !== undefined) {
|
|
payload.last_seen_days = lastSeen;
|
|
}
|
|
|
|
const firstSeen = parseInteger(values.first_seen_days);
|
|
if (firstSeen !== undefined) {
|
|
payload.first_seen_days = firstSeen;
|
|
}
|
|
|
|
const pageSize = parseInteger(values.page_size);
|
|
if (pageSize !== undefined) {
|
|
payload.page_size = pageSize;
|
|
}
|
|
|
|
const maxPages = parseInteger(values.max_pages);
|
|
if (maxPages !== undefined) {
|
|
payload.max_pages = maxPages;
|
|
}
|
|
|
|
payload.enabled = values.enabled ? 1 : 0;
|
|
payload.enrich_llm = values.enrich_llm ? 1 : 0;
|
|
payload.only_match = values.only_match ? 1 : 0;
|
|
|
|
// Ne pas envoyer 'once' si non coché
|
|
if (values.once) {
|
|
payload.once = 1;
|
|
}
|
|
|
|
return payload;
|
|
}
|
|
|
|
function createDefaultScraperValues(): ScraperFormValues {
|
|
return {
|
|
params: buildDefaultValue(scraperSchema) as JsonObject,
|
|
frequency: "",
|
|
task_name: "",
|
|
property_types: "",
|
|
last_seen_days: "",
|
|
first_seen_days: "",
|
|
page_size: "",
|
|
max_pages: "",
|
|
enabled: true,
|
|
enrich_llm: false,
|
|
only_match: false,
|
|
once: false,
|
|
};
|
|
}
|
|
|
|
function formatParamsDisplay(params: string | null) {
|
|
if (!params) {
|
|
return null;
|
|
}
|
|
|
|
try {
|
|
return JSON.stringify(JSON.parse(params), null, 2);
|
|
} catch (parseError) {
|
|
return params;
|
|
}
|
|
}
|
|
|
|
export function ScrapersTab() {
|
|
const [paramsValid, setParamsValid] = useState(true);
|
|
const [editingScraperId, setEditingScraperId] = useState<string | null>(null);
|
|
const [statusMessage, setStatusMessage] = useState<string | null>(null);
|
|
const [isCountDialogOpen, setIsCountDialogOpen] = useState(false);
|
|
const [countFirstSeenDate, setCountFirstSeenDate] = useState<Date | undefined>(undefined);
|
|
const [countLastSeenDate, setCountLastSeenDate] = useState<Date | undefined>(undefined);
|
|
const [isAiDialogOpen, setIsAiDialogOpen] = useState(false);
|
|
const [aiPrompt, setAiPrompt] = useState("");
|
|
const queryClient = useQueryClient();
|
|
const initialValues = useMemo(() => createDefaultScraperValues(), []);
|
|
const form = useForm<ScraperFormValues>({
|
|
defaultValues: initialValues,
|
|
mode: "onBlur",
|
|
});
|
|
|
|
const scrapersQuery = useQuery({
|
|
queryKey: ["scrapers"],
|
|
queryFn: () => api.get<Scraper[]>("/scrapers"),
|
|
});
|
|
|
|
const createScraperMutation = useMutation({
|
|
mutationFn: (values: ScraperFormValues) =>
|
|
api.post<CreateScraperPayload, Scraper>(
|
|
"/scrapers",
|
|
transformPayload(values),
|
|
),
|
|
onSuccess: () => {
|
|
queryClient.invalidateQueries({ queryKey: ["scrapers"] });
|
|
setParamsValid(true);
|
|
setEditingScraperId(null);
|
|
form.reset(createDefaultScraperValues());
|
|
setStatusMessage("Scraper cree avec succes.");
|
|
},
|
|
});
|
|
|
|
const updateScraperMutation = useMutation({
|
|
mutationFn: ({ id, data }: { id: string; data: ScraperFormValues }) => {
|
|
const payload = transformPayload(data);
|
|
return api.put<CreateScraperPayload, Scraper>(`/scrapers/${id}`, payload);
|
|
},
|
|
onSuccess: () => {
|
|
queryClient.invalidateQueries({ queryKey: ["scrapers"] });
|
|
setEditingScraperId(null);
|
|
setParamsValid(true);
|
|
form.reset(createDefaultScraperValues());
|
|
setStatusMessage("Scraper mis a jour.");
|
|
},
|
|
});
|
|
|
|
const deleteScraperMutation = useMutation({
|
|
mutationFn: (id: string) => api.delete(`/scrapers/${id}`),
|
|
onSuccess: (_, id) => {
|
|
if (editingScraperId === id) {
|
|
setEditingScraperId(null);
|
|
setParamsValid(true);
|
|
form.reset(createDefaultScraperValues());
|
|
}
|
|
queryClient.invalidateQueries({ queryKey: ["scrapers"] });
|
|
setStatusMessage("Scraper supprime.");
|
|
},
|
|
});
|
|
|
|
type ScraperCountResponse = { count: number };
|
|
|
|
const countScraperMutation = useMutation({
|
|
mutationFn: (payload: { values: ScraperFormValues; firstSeenDate?: Date; lastSeenDate?: Date }) => {
|
|
const basePayload = transformPayload(payload.values);
|
|
|
|
// Override first_seen_days and last_seen_days with date-based calculations if dates are provided
|
|
if (payload.firstSeenDate) {
|
|
const daysSinceFirstSeen = Math.floor((new Date().getTime() - payload.firstSeenDate.getTime()) / (1000 * 60 * 60 * 24));
|
|
basePayload.first_seen_days = daysSinceFirstSeen;
|
|
}
|
|
|
|
if (payload.lastSeenDate) {
|
|
const daysSinceLastSeen = Math.floor((new Date().getTime() - payload.lastSeenDate.getTime()) / (1000 * 60 * 60 * 24));
|
|
basePayload.last_seen_days = daysSinceLastSeen;
|
|
}
|
|
|
|
console.log("Count payload being sent:", basePayload);
|
|
|
|
return api.post<CreateScraperPayload, ScraperCountResponse>(
|
|
"/scrapers/count",
|
|
basePayload,
|
|
);
|
|
},
|
|
});
|
|
|
|
type AiGenerateResponse = { params: JsonObject };
|
|
|
|
const aiGenerateMutation = useMutation({
|
|
mutationFn: (prompt: string) =>
|
|
api.post<{ prompt: string }, AiGenerateResponse>("/ai/generate-scraper", { prompt }),
|
|
onSuccess: (data) => {
|
|
if (data.params) {
|
|
form.setValue("params", data.params);
|
|
setStatusMessage("Scraper genere par IA avec succes.");
|
|
setIsAiDialogOpen(false);
|
|
setAiPrompt("");
|
|
}
|
|
},
|
|
});
|
|
|
|
const handleAiGenerate = () => {
|
|
if (aiPrompt.trim()) {
|
|
aiGenerateMutation.mutate(aiPrompt.trim());
|
|
}
|
|
};
|
|
|
|
const onSubmit = form.handleSubmit(async (values) => {
|
|
setStatusMessage(null);
|
|
if (editingScraperId) {
|
|
await updateScraperMutation.mutateAsync({
|
|
id: editingScraperId,
|
|
data: values,
|
|
});
|
|
} else {
|
|
await createScraperMutation.mutateAsync(values);
|
|
}
|
|
});
|
|
|
|
const formError =
|
|
(createScraperMutation.isError &&
|
|
getErrorMessage(createScraperMutation.error)) ||
|
|
(updateScraperMutation.isError &&
|
|
getErrorMessage(updateScraperMutation.error)) ||
|
|
null;
|
|
|
|
const handleOpenCountDialog = async () => {
|
|
setStatusMessage(null);
|
|
const isFormValid = await form.trigger();
|
|
|
|
if (!isFormValid || !paramsValid) {
|
|
return;
|
|
}
|
|
|
|
countScraperMutation.reset();
|
|
setCountFirstSeenDate(undefined);
|
|
setCountLastSeenDate(undefined);
|
|
setIsCountDialogOpen(true);
|
|
};
|
|
|
|
const handleCountWithDates = () => {
|
|
countScraperMutation.mutate({
|
|
values: form.getValues(),
|
|
firstSeenDate: countFirstSeenDate,
|
|
lastSeenDate: countLastSeenDate,
|
|
});
|
|
};
|
|
|
|
const scrapers = useMemo(
|
|
() => scrapersQuery.data ?? [],
|
|
[scrapersQuery.data],
|
|
);
|
|
|
|
const scrollToTop = () => {
|
|
window.scrollTo({ top: 0, behavior: "instant" });
|
|
};
|
|
|
|
const scrollToList = () => {
|
|
const listElement = document.getElementById("scrapers-list");
|
|
if (listElement) {
|
|
listElement.scrollIntoView({ behavior: "instant" });
|
|
}
|
|
};
|
|
|
|
return (
|
|
<div className="space-y-6">
|
|
<div className="fixed bottom-6 right-6 flex flex-col gap-2 z-50">
|
|
<Button
|
|
type="button"
|
|
size="icon"
|
|
variant="secondary"
|
|
className="rounded-full shadow-lg"
|
|
onClick={scrollToList}
|
|
title="Aller à la liste"
|
|
>
|
|
<ArrowDown className="h-4 w-4" />
|
|
</Button>
|
|
<Button
|
|
type="button"
|
|
size="icon"
|
|
variant="secondary"
|
|
className="rounded-full shadow-lg"
|
|
onClick={scrollToTop}
|
|
title="Retour en haut"
|
|
>
|
|
<ArrowUp className="h-4 w-4" />
|
|
</Button>
|
|
</div>
|
|
|
|
<Card>
|
|
<CardHeader>
|
|
<div className="flex items-center justify-between">
|
|
<div>
|
|
<CardTitle>Creer un scraper</CardTitle>
|
|
<CardDescription>
|
|
Definissez la configuration et les parametres d'execution du
|
|
scraper.
|
|
</CardDescription>
|
|
</div>
|
|
<Button
|
|
type="button"
|
|
variant="outline"
|
|
onClick={() => setIsAiDialogOpen(true)}
|
|
>
|
|
<Sparkles className="mr-2 h-4 w-4" />
|
|
Generer avec IA
|
|
</Button>
|
|
</div>
|
|
</CardHeader>
|
|
<CardContent>
|
|
<form className="space-y-6" onSubmit={onSubmit}>
|
|
<div className="grid gap-4 md:grid-cols-2">
|
|
<div className="space-y-1 md:col-span-2">
|
|
<label className="text-sm font-medium" htmlFor="task_name">
|
|
Nom du scraper
|
|
</label>
|
|
<p className="text-xs text-muted-foreground">
|
|
Identifiant unique pour reperer ce scraper
|
|
</p>
|
|
<Input
|
|
id="task_name"
|
|
placeholder="Nom interne"
|
|
{...form.register("task_name")}
|
|
/>
|
|
</div>
|
|
|
|
<div className="space-y-1">
|
|
<label className="text-sm font-medium" htmlFor="frequency">
|
|
Frequence
|
|
</label>
|
|
<p className="text-xs text-muted-foreground">
|
|
Expression cron pour planifier l'execution
|
|
</p>
|
|
<Input
|
|
id="frequency"
|
|
placeholder="0 7 * * *"
|
|
{...form.register("frequency")}
|
|
/>
|
|
</div>
|
|
|
|
<div className="space-y-1">
|
|
<label className="text-sm font-medium" htmlFor="property_types">
|
|
Types de biens
|
|
</label>
|
|
<p className="text-xs text-muted-foreground">
|
|
Types de proprietes a rechercher (separes par virgule)
|
|
</p>
|
|
<Input
|
|
id="property_types"
|
|
placeholder="maison,appartement,immeuble"
|
|
aria-invalid={!!form.formState.errors.property_types}
|
|
{...form.register("property_types", {
|
|
validate: (value) => {
|
|
if (!value || value.trim() === "") {
|
|
return true; // Champ optionnel
|
|
}
|
|
const validTypes = ["immeuble", "appartement", "maison"];
|
|
const types = value
|
|
.split(",")
|
|
.map((t) => t.trim().toLowerCase())
|
|
.filter((t) => t !== "");
|
|
|
|
const invalidTypes = types.filter(
|
|
(t) => !validTypes.includes(t)
|
|
);
|
|
|
|
if (invalidTypes.length > 0) {
|
|
return `Types invalides: ${invalidTypes.join(", ")}. Valeurs autorisées: ${validTypes.join(", ")}`;
|
|
}
|
|
|
|
return true;
|
|
},
|
|
})}
|
|
/>
|
|
{form.formState.errors.property_types && (
|
|
<p className="text-destructive text-xs">
|
|
{form.formState.errors.property_types.message}
|
|
</p>
|
|
)}
|
|
</div>
|
|
|
|
<div className="grid gap-3 md:grid-cols-2">
|
|
<div className="space-y-1">
|
|
<label
|
|
className="text-sm font-medium"
|
|
htmlFor="last_seen_days"
|
|
>
|
|
Derniere vue (jours)
|
|
</label>
|
|
<p className="text-xs text-muted-foreground">
|
|
Annonces vues il y a maximum N jours
|
|
</p>
|
|
<Input
|
|
id="last_seen_days"
|
|
type="number"
|
|
min="0"
|
|
{...form.register("last_seen_days")}
|
|
/>
|
|
</div>
|
|
|
|
<div className="space-y-1">
|
|
<label
|
|
className="text-sm font-medium"
|
|
htmlFor="first_seen_days"
|
|
>
|
|
Premiere vue (jours)
|
|
</label>
|
|
<p className="text-xs text-muted-foreground">
|
|
Annonces apparues il y a minimum N jours
|
|
</p>
|
|
<Input
|
|
id="first_seen_days"
|
|
type="number"
|
|
min="0"
|
|
{...form.register("first_seen_days")}
|
|
/>
|
|
</div>
|
|
|
|
<div className="space-y-1">
|
|
<label className="text-sm font-medium" htmlFor="page_size">
|
|
Taille de page
|
|
</label>
|
|
<p className="text-xs text-muted-foreground">
|
|
Nombre de resultats par page
|
|
</p>
|
|
<Input
|
|
id="page_size"
|
|
type="number"
|
|
min="1"
|
|
{...form.register("page_size")}
|
|
/>
|
|
</div>
|
|
|
|
<div className="space-y-1">
|
|
<label className="text-sm font-medium" htmlFor="max_pages">
|
|
Pages maximum
|
|
</label>
|
|
<p className="text-xs text-muted-foreground">
|
|
Nombre maximum de pages a parcourir
|
|
</p>
|
|
<Input
|
|
id="max_pages"
|
|
type="number"
|
|
min="1"
|
|
{...form.register("max_pages")}
|
|
/>
|
|
</div>
|
|
</div>
|
|
|
|
<div className="grid gap-4 md:col-span-2 md:grid-cols-2">
|
|
<Controller
|
|
control={form.control}
|
|
name="enabled"
|
|
render={({ field }) => (
|
|
<div className="space-y-1">
|
|
<label className="flex items-center gap-2 text-sm font-medium">
|
|
<Checkbox
|
|
checked={field.value}
|
|
onCheckedChange={(checked) =>
|
|
field.onChange(Boolean(checked))
|
|
}
|
|
/>
|
|
Activer
|
|
</label>
|
|
<p className="text-xs text-muted-foreground ml-6">
|
|
Active ou desactive l'execution automatique
|
|
</p>
|
|
</div>
|
|
)}
|
|
/>
|
|
|
|
<Controller
|
|
control={form.control}
|
|
name="enrich_llm"
|
|
render={({ field }) => (
|
|
<div className="space-y-1">
|
|
<label className="flex items-center gap-2 text-sm font-medium">
|
|
<Checkbox
|
|
checked={field.value}
|
|
onCheckedChange={(checked) =>
|
|
field.onChange(Boolean(checked))
|
|
}
|
|
/>
|
|
Enrichir via LLM
|
|
</label>
|
|
<p className="text-xs text-muted-foreground ml-6">
|
|
Analyse les annonces avec un modele de langage
|
|
</p>
|
|
</div>
|
|
)}
|
|
/>
|
|
|
|
<Controller
|
|
control={form.control}
|
|
name="only_match"
|
|
render={({ field }) => (
|
|
<div className="space-y-1">
|
|
<label className="flex items-center gap-2 text-sm font-medium">
|
|
<Checkbox
|
|
checked={field.value}
|
|
onCheckedChange={(checked) =>
|
|
field.onChange(Boolean(checked))
|
|
}
|
|
/>
|
|
Mode strict
|
|
</label>
|
|
<p className="text-xs text-muted-foreground ml-6">
|
|
Ne retourne que les correspondances exactes
|
|
</p>
|
|
</div>
|
|
)}
|
|
/>
|
|
|
|
<Controller
|
|
control={form.control}
|
|
name="once"
|
|
render={({ field }) => (
|
|
<div className="space-y-1">
|
|
<label className="flex items-center gap-2 text-sm font-medium">
|
|
<Checkbox
|
|
checked={field.value}
|
|
onCheckedChange={(checked) =>
|
|
field.onChange(Boolean(checked))
|
|
}
|
|
/>
|
|
Une fois
|
|
</label>
|
|
<p className="text-xs text-muted-foreground ml-6">
|
|
Execute le scraper une seule fois
|
|
</p>
|
|
</div>
|
|
)}
|
|
/>
|
|
</div>
|
|
</div>
|
|
|
|
<Controller
|
|
control={form.control}
|
|
name="params"
|
|
render={({ field }) => (
|
|
<SchemaAwareJsonField
|
|
schema={scraperSchema}
|
|
value={field.value}
|
|
onChange={(next) => field.onChange(next)}
|
|
label="Parametres"
|
|
description="Structure schemaisee conformement au schema scraper."
|
|
onValidationChange={(payload) =>
|
|
setParamsValid(payload.valid)
|
|
}
|
|
/>
|
|
)}
|
|
/>
|
|
|
|
<div className="flex flex-wrap items-center gap-3">
|
|
<Button
|
|
type="submit"
|
|
disabled={
|
|
(editingScraperId
|
|
? updateScraperMutation.isPending
|
|
: createScraperMutation.isPending) || !paramsValid
|
|
}
|
|
>
|
|
{editingScraperId
|
|
? updateScraperMutation.isPending
|
|
? "Mise a jour..."
|
|
: "Mettre a jour le scraper"
|
|
: createScraperMutation.isPending
|
|
? "Creation..."
|
|
: "Creer le scraper"}
|
|
</Button>
|
|
|
|
{statusMessage ? (
|
|
<p className="text-emerald-600 text-sm">{statusMessage}</p>
|
|
) : null}
|
|
|
|
{formError ? (
|
|
<p className="text-destructive text-sm">{formError}</p>
|
|
) : null}
|
|
|
|
{!paramsValid ? (
|
|
<p className="text-destructive text-sm">
|
|
Corrigez les erreurs du schema avant l'envoi.
|
|
</p>
|
|
) : null}
|
|
|
|
<Button
|
|
type="button"
|
|
variant="secondary"
|
|
disabled={countScraperMutation.isPending}
|
|
onClick={handleOpenCountDialog}
|
|
>
|
|
{countScraperMutation.isPending
|
|
? "Calcul..."
|
|
: "Compter les resultats"}
|
|
</Button>
|
|
|
|
{editingScraperId ? (
|
|
<Button
|
|
type="button"
|
|
variant="outline"
|
|
onClick={() => {
|
|
setStatusMessage(null);
|
|
setEditingScraperId(null);
|
|
setParamsValid(true);
|
|
form.reset(createDefaultScraperValues());
|
|
}}
|
|
>
|
|
Annuler la modification
|
|
</Button>
|
|
) : null}
|
|
</div>
|
|
</form>
|
|
</CardContent>
|
|
</Card>
|
|
|
|
<Card id="scrapers-list">
|
|
<CardHeader>
|
|
<div className="flex items-center justify-between">
|
|
<div>
|
|
<CardTitle>Scrapers existants</CardTitle>
|
|
<CardDescription>
|
|
{scrapersQuery.isLoading
|
|
? "Chargement des scrapers..."
|
|
: scrapers.length === 0
|
|
? "Aucun scraper enregistre."
|
|
: "Liste recue depuis l'API."}
|
|
</CardDescription>
|
|
</div>
|
|
{scrapers.length > 0 && (
|
|
<Button
|
|
type="button"
|
|
variant="outline"
|
|
size="sm"
|
|
onClick={() => {
|
|
navigator.clipboard.writeText(JSON.stringify(scrapers, null, 2));
|
|
setStatusMessage("Scrapers copies dans le presse-papier!");
|
|
setTimeout(() => setStatusMessage(null), 3000);
|
|
}}
|
|
>
|
|
Copier tous (JSON)
|
|
</Button>
|
|
)}
|
|
</div>
|
|
</CardHeader>
|
|
<CardContent className="overflow-x-auto">
|
|
{scrapersQuery.isError ? (
|
|
<p className="text-destructive text-sm">
|
|
{getErrorMessage(scrapersQuery.error)}
|
|
</p>
|
|
) : (
|
|
<table className="min-w-full table-fixed border-collapse text-sm">
|
|
<thead>
|
|
<tr className="text-left">
|
|
<th className="border-b px-3 py-2 font-semibold">Nom</th>
|
|
<th className="border-b px-3 py-2 font-semibold">
|
|
Frequence
|
|
</th>
|
|
<th className="border-b px-3 py-2 font-semibold">
|
|
Parametres
|
|
</th>
|
|
<th className="border-b px-3 py-2 font-semibold">Etat</th>
|
|
<th className="border-b px-3 py-2 font-semibold">Limites</th>
|
|
<th className="border-b px-3 py-2 font-semibold">Actions</th>
|
|
</tr>
|
|
</thead>
|
|
<tbody>
|
|
{scrapers.map((scraper) => {
|
|
const paramsDisplay = formatParamsDisplay(scraper.params);
|
|
|
|
return (
|
|
<tr key={scraper.id} className="align-top">
|
|
<td className="border-b px-3 py-2 font-medium">
|
|
{scraper.task_name ?? "-"}
|
|
</td>
|
|
<td className="border-b px-3 py-2 text-muted-foreground">
|
|
{scraper.frequency ?? "-"}
|
|
</td>
|
|
<td className="border-b px-3 py-2">
|
|
<div className="space-y-2">
|
|
{paramsDisplay ? (
|
|
<pre className="max-h-40 overflow-auto rounded bg-muted/50 p-2 text-xs">
|
|
{paramsDisplay}
|
|
</pre>
|
|
) : (
|
|
<span className="text-muted-foreground">-</span>
|
|
)}
|
|
{scraper.params && (
|
|
<Button
|
|
type="button"
|
|
variant="outline"
|
|
size="sm"
|
|
onClick={() => {
|
|
navigator.clipboard.writeText(scraper.params || "");
|
|
setStatusMessage("JSON copié dans le presse-papier!");
|
|
setTimeout(() => setStatusMessage(null), 3000);
|
|
}}
|
|
>
|
|
Copier JSON
|
|
</Button>
|
|
)}
|
|
</div>
|
|
</td>
|
|
<td className="border-b px-3 py-2">
|
|
<div className="space-y-1 text-xs">
|
|
<span
|
|
className={cn(
|
|
"inline-flex items-center gap-1 rounded-full px-2 py-0.5 font-medium",
|
|
scraper.enabled
|
|
? "bg-emerald-100 text-emerald-700"
|
|
: "bg-gray-200 text-gray-600",
|
|
)}
|
|
>
|
|
Actif: {scraper.enabled ? "Oui" : "Non"}
|
|
</span>
|
|
<span className="block text-muted-foreground">
|
|
LLM: {scraper.enrich_llm ? "Oui" : "Non"}
|
|
</span>
|
|
<span className="block text-muted-foreground">
|
|
Strict: {scraper.only_match ? "Oui" : "Non"}
|
|
</span>
|
|
<span className="block text-muted-foreground">
|
|
Une fois: {scraper.once ? "Oui" : "Non"}
|
|
</span>
|
|
</div>
|
|
</td>
|
|
<td className="border-b px-3 py-2 text-xs text-muted-foreground">
|
|
<div className="space-y-1">
|
|
<span>
|
|
Derniere vue: {scraper.last_seen_days ?? "-"}
|
|
</span>
|
|
<span>
|
|
Premiere vue: {scraper.first_seen_days ?? "-"}
|
|
</span>
|
|
<span>Page size: {scraper.page_size ?? "-"}</span>
|
|
<span>Max pages: {scraper.max_pages ?? "-"}</span>
|
|
</div>
|
|
</td>
|
|
<td className="border-b px-3 py-2">
|
|
<div className="flex flex-wrap gap-2">
|
|
<Button
|
|
type="button"
|
|
variant="outline"
|
|
size="sm"
|
|
onClick={() => {
|
|
setStatusMessage(null);
|
|
setEditingScraperId(scraper.id);
|
|
setParamsValid(true);
|
|
let parsedParams: JsonObject = buildDefaultValue(
|
|
scraperSchema,
|
|
) as JsonObject;
|
|
if (scraper.params) {
|
|
try {
|
|
const raw = JSON.parse(
|
|
scraper.params,
|
|
) as JsonObject;
|
|
parsedParams =
|
|
raw && typeof raw === "object"
|
|
? raw
|
|
: parsedParams;
|
|
} catch (error) {
|
|
parsedParams = buildDefaultValue(
|
|
scraperSchema,
|
|
) as JsonObject;
|
|
}
|
|
}
|
|
form.reset({
|
|
params: parsedParams,
|
|
frequency: scraper.frequency ?? "",
|
|
task_name: scraper.task_name ?? "",
|
|
property_types: scraper.property_types ?? "",
|
|
last_seen_days:
|
|
scraper.last_seen_days?.toString() ?? "",
|
|
first_seen_days:
|
|
scraper.first_seen_days?.toString() ?? "",
|
|
page_size: scraper.page_size?.toString() ?? "",
|
|
max_pages: scraper.max_pages?.toString() ?? "",
|
|
enabled: Boolean(scraper.enabled),
|
|
enrich_llm: Boolean(scraper.enrich_llm),
|
|
only_match: Boolean(scraper.only_match),
|
|
once: Boolean(scraper.once),
|
|
});
|
|
}}
|
|
>
|
|
Modifier
|
|
</Button>
|
|
<Button
|
|
type="button"
|
|
variant="destructive"
|
|
size="sm"
|
|
disabled={deleteScraperMutation.isPending}
|
|
onClick={() => {
|
|
if (
|
|
window.confirm(
|
|
"Confirmez la suppression de ce scraper ?",
|
|
)
|
|
) {
|
|
deleteScraperMutation.mutate(scraper.id);
|
|
}
|
|
}}
|
|
>
|
|
Supprimer
|
|
</Button>
|
|
</div>
|
|
</td>
|
|
</tr>
|
|
);
|
|
})}
|
|
</tbody>
|
|
</table>
|
|
)}
|
|
</CardContent>
|
|
</Card>
|
|
|
|
<Dialog
|
|
open={isCountDialogOpen}
|
|
onOpenChange={(open) => {
|
|
setIsCountDialogOpen(open);
|
|
if (!open) {
|
|
countScraperMutation.reset();
|
|
setCountFirstSeenDate(undefined);
|
|
setCountLastSeenDate(undefined);
|
|
}
|
|
}}
|
|
>
|
|
<DialogContent className="max-w-2xl">
|
|
<DialogHeader>
|
|
<DialogTitle>Estimation du nombre d'annonces</DialogTitle>
|
|
<DialogDescription>
|
|
Calcul base sur la configuration actuelle du scraper. Vous pouvez filtrer par période.
|
|
</DialogDescription>
|
|
</DialogHeader>
|
|
|
|
<div className="space-y-4">
|
|
<div className="grid grid-cols-2 gap-4">
|
|
<div className="space-y-2">
|
|
<label className="text-sm font-medium">Date de première vue (min)</label>
|
|
<Popover>
|
|
<PopoverTrigger asChild>
|
|
<Button
|
|
variant="outline"
|
|
className={cn(
|
|
"w-full justify-start text-left font-normal",
|
|
!countFirstSeenDate && "text-muted-foreground"
|
|
)}
|
|
>
|
|
<CalendarIcon className="mr-2 h-4 w-4" />
|
|
{countFirstSeenDate ? format(countFirstSeenDate, "PPP") : "Sélectionner une date"}
|
|
</Button>
|
|
</PopoverTrigger>
|
|
<PopoverContent className="w-auto p-0">
|
|
<Calendar
|
|
mode="single"
|
|
selected={countFirstSeenDate}
|
|
onSelect={setCountFirstSeenDate}
|
|
initialFocus
|
|
/>
|
|
</PopoverContent>
|
|
</Popover>
|
|
<p className="text-xs text-muted-foreground">
|
|
Annonces vues pour la première fois après cette date
|
|
</p>
|
|
</div>
|
|
|
|
<div className="space-y-2">
|
|
<label className="text-sm font-medium">Date de dernière vue (max)</label>
|
|
<Popover>
|
|
<PopoverTrigger asChild>
|
|
<Button
|
|
variant="outline"
|
|
className={cn(
|
|
"w-full justify-start text-left font-normal",
|
|
!countLastSeenDate && "text-muted-foreground"
|
|
)}
|
|
>
|
|
<CalendarIcon className="mr-2 h-4 w-4" />
|
|
{countLastSeenDate ? format(countLastSeenDate, "PPP") : "Sélectionner une date"}
|
|
</Button>
|
|
</PopoverTrigger>
|
|
<PopoverContent className="w-auto p-0">
|
|
<Calendar
|
|
mode="single"
|
|
selected={countLastSeenDate}
|
|
onSelect={setCountLastSeenDate}
|
|
initialFocus
|
|
/>
|
|
</PopoverContent>
|
|
</Popover>
|
|
<p className="text-xs text-muted-foreground">
|
|
Annonces vues pour la dernière fois avant cette date
|
|
</p>
|
|
</div>
|
|
</div>
|
|
|
|
<Button
|
|
onClick={handleCountWithDates}
|
|
disabled={countScraperMutation.isPending}
|
|
className="w-full"
|
|
>
|
|
{countScraperMutation.isPending ? "Calcul en cours..." : "Calculer le nombre d'annonces"}
|
|
</Button>
|
|
|
|
<div className="min-h-[60px] rounded-md border p-4">
|
|
{countScraperMutation.isPending ? (
|
|
<p className="text-sm text-muted-foreground">
|
|
Calcul en cours...
|
|
</p>
|
|
) : countScraperMutation.isError ? (
|
|
<p className="text-destructive text-sm">
|
|
{getErrorMessage(countScraperMutation.error)}
|
|
</p>
|
|
) : countScraperMutation.data ? (
|
|
<p className="text-sm">
|
|
Ce scraper correspond a
|
|
<span className="font-semibold text-lg">
|
|
{" "}
|
|
{countScraperMutation.data.count}{" "}
|
|
</span>
|
|
annonce(s) potentielle(s).
|
|
</p>
|
|
) : (
|
|
<p className="text-muted-foreground text-sm">
|
|
Cliquez sur "Calculer" pour obtenir le nombre d'annonces.
|
|
</p>
|
|
)}
|
|
</div>
|
|
</div>
|
|
|
|
<DialogFooter>
|
|
<DialogClose asChild>
|
|
<Button variant="outline">Fermer</Button>
|
|
</DialogClose>
|
|
</DialogFooter>
|
|
</DialogContent>
|
|
</Dialog>
|
|
|
|
<Dialog
|
|
open={isAiDialogOpen}
|
|
onOpenChange={(open) => {
|
|
setIsAiDialogOpen(open);
|
|
if (!open) {
|
|
aiGenerateMutation.reset();
|
|
setAiPrompt("");
|
|
}
|
|
}}
|
|
>
|
|
<DialogContent className="max-w-2xl">
|
|
<DialogHeader>
|
|
<DialogTitle>Generer un scraper avec l'IA</DialogTitle>
|
|
<DialogDescription>
|
|
Decrivez en langage naturel le type de scraper que vous souhaitez creer.
|
|
L'IA generera les parametres JSON correspondants.
|
|
</DialogDescription>
|
|
</DialogHeader>
|
|
|
|
<div className="space-y-4">
|
|
<Textarea
|
|
placeholder="Ex: Je veux un scraper pour les maisons en Bretagne (departements 22, 29, 35, 56) a acheter, avec un prix maximum de 200000 euros..."
|
|
value={aiPrompt}
|
|
onChange={(e) => setAiPrompt(e.target.value)}
|
|
rows={5}
|
|
className="resize-none"
|
|
/>
|
|
|
|
<Button
|
|
onClick={handleAiGenerate}
|
|
disabled={aiGenerateMutation.isPending || !aiPrompt.trim()}
|
|
className="w-full"
|
|
>
|
|
<Sparkles className="mr-2 h-4 w-4" />
|
|
{aiGenerateMutation.isPending ? "Generation en cours..." : "Generer le scraper"}
|
|
</Button>
|
|
|
|
{aiGenerateMutation.isError && (
|
|
<p className="text-destructive text-sm">
|
|
{getErrorMessage(aiGenerateMutation.error)}
|
|
</p>
|
|
)}
|
|
</div>
|
|
|
|
<DialogFooter>
|
|
<DialogClose asChild>
|
|
<Button variant="outline">Annuler</Button>
|
|
</DialogClose>
|
|
</DialogFooter>
|
|
</DialogContent>
|
|
</Dialog>
|
|
</div>
|
|
);
|
|
}
|