count thing added
This commit is contained in:
Vitrixxl
parent
94f5d37f7a
commit
ef417be648
@ -10,6 +10,11 @@
|
||||
"allow_origin": "*",
|
||||
"allow_credentials": true,
|
||||
"notes": "Server responds to OPTIONS preflight without authentication and accepts credentialed requests from any origin."
|
||||
},
|
||||
"environment": {
|
||||
"API_TOKEN": "Bearer token expected on every request.",
|
||||
"FLUXIMMO_API_KEY": "Credential forwarded to Fluximmo analytics count endpoint.",
|
||||
"FLUXIMMO_COUNT_URL": "Optional override for Fluximmo count API base URL (defaults to https://analytics.fluximmo.io/properties/count)."
|
||||
}
|
||||
},
|
||||
"authentication": {
|
||||
@ -37,7 +42,7 @@
|
||||
"Profile": {
|
||||
"description": "Investment profile descriptor persisted in users_investmentprofile.",
|
||||
"fields": [
|
||||
{"name": "profile_id", "type": "uuid", "nullable": false, "read_only": false, "description": "Primary key. Optional on create; autogenerated when omitted."},
|
||||
{"name": "profile_id", "type": "uuid", "nullable": false, "read_only": true, "description": "Primary key generated server-side."},
|
||||
{"name": "name", "type": "string", "nullable": false, "read_only": false, "description": "Human-readable profile label."},
|
||||
{"name": "description", "type": "string", "nullable": true, "read_only": false, "description": "Optional free-form description. Empty string preserved."},
|
||||
{"name": "criteria", "type": "json", "nullable": false, "read_only": false, "description": "Arbitrary JSON object/array describing selection rules."},
|
||||
@ -64,7 +69,7 @@
|
||||
"Scraper": {
|
||||
"description": "Scraper configuration persisted in scraper table.",
|
||||
"fields": [
|
||||
{"name": "id", "type": "string", "nullable": false, "read_only": false, "description": "Primary key identifier."},
|
||||
{"name": "id", "type": "uuid", "nullable": false, "read_only": true, "description": "Primary key generated server-side."},
|
||||
{"name": "params", "type": "string", "nullable": true, "read_only": false, "description": "Serialized scraper parameters."},
|
||||
{"name": "last_seen_days", "type": "integer", "nullable": true, "read_only": false, "description": "Number of days since listings were last seen."},
|
||||
{"name": "first_seen_days", "type": "integer", "nullable": true, "read_only": false, "description": "Number of days since listings were first seen."},
|
||||
@ -133,7 +138,6 @@
|
||||
"type": "object",
|
||||
"required_fields": ["name", "criteria"],
|
||||
"fields": {
|
||||
"profile_id": {"type": "uuid", "nullable": false, "description": "Optional explicit UUID. Auto-generated when omitted."},
|
||||
"name": {"type": "string", "nullable": false, "description": "Profile name."},
|
||||
"description": {"type": "string", "nullable": true, "description": "Optional description. Use null to clear."},
|
||||
"criteria": {"type": "json", "nullable": false, "description": "Required JSON criteria payload."},
|
||||
@ -394,9 +398,8 @@
|
||||
},
|
||||
"body": {
|
||||
"type": "object",
|
||||
"required_fields": ["id"],
|
||||
"required_fields": [],
|
||||
"fields": {
|
||||
"id": {"type": "string", "nullable": false, "description": "Scraper primary key."},
|
||||
"params": {"type": "string", "nullable": true, "description": "Serialized parameter payload."},
|
||||
"frequency": {"type": "string", "nullable": true, "description": "Execution cadence descriptor."},
|
||||
"task_name": {"type": "string", "nullable": true, "description": "Linked task name."},
|
||||
@ -481,6 +484,34 @@
|
||||
"404": {"description": "Scraper not found."},
|
||||
"503": {"description": "Database unavailable."}
|
||||
}
|
||||
},
|
||||
{
|
||||
"operation_id": "countScraperProperties",
|
||||
"method": "POST",
|
||||
"path": "/scrapers/count",
|
||||
"authenticated": true,
|
||||
"description": "Return the Fluximmo property count derived from a scraper configuration.",
|
||||
"request": {
|
||||
"headers": {
|
||||
"Content-Type": {"type": "string", "required": true, "example": "application/json"},
|
||||
"Authorization": {"type": "string", "required": true, "description": "Bearer <API_TOKEN>"}
|
||||
},
|
||||
"body": {
|
||||
"type": "object",
|
||||
"required_fields": ["params"],
|
||||
"fields": {
|
||||
"params": {"type": "object", "nullable": false, "description": "Base Fluximmo search filters. Accepts a JSON object or string-encoded JSON."},
|
||||
"first_seen_days": {"type": "integer", "nullable": true, "description": "Days before now for firstSeenAt.min."},
|
||||
"last_seen_days": {"type": "integer", "nullable": true, "description": "Days before now for lastSeenAt.max."}
|
||||
}
|
||||
}
|
||||
},
|
||||
"responses": {
|
||||
"200": {"description": "Count retrieved.", "body": {"type": "object", "fields": {"count": {"type": "integer", "nullable": false}}}},
|
||||
"400": {"description": "Validation failure."},
|
||||
"401": {"description": "Missing or invalid bearer token."},
|
||||
"502": {"description": "Fluximmo upstream error."}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
136
backend/app.py
136
backend/app.py
@ -3,7 +3,9 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from datetime import datetime, timezone
|
||||
import json
|
||||
from copy import deepcopy
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any, Dict, Iterable, List, Mapping, MutableMapping
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
@ -11,13 +13,20 @@ import psycopg
|
||||
from psycopg import sql
|
||||
from psycopg.rows import dict_row
|
||||
from psycopg.types.json import Json
|
||||
from deepmerge import always_merger
|
||||
from dotenv import load_dotenv
|
||||
from flask import Flask, abort, jsonify, request, g
|
||||
from flask_cors import CORS
|
||||
import requests
|
||||
|
||||
load_dotenv()
|
||||
|
||||
API_TOKEN = os.getenv("API_TOKEN")
|
||||
FLUXIMMO_API_KEY = os.getenv("FLUXIMMO_API_KEY")
|
||||
FLUXIMMO_COUNT_URL = os.getenv(
|
||||
"FLUXIMMO_COUNT_URL",
|
||||
"https://analytics.fluximmo.io/properties/count",
|
||||
)
|
||||
|
||||
|
||||
REQUIRED_DB_SETTINGS = {
|
||||
@ -57,6 +66,10 @@ if not API_TOKEN:
|
||||
raise RuntimeError(
|
||||
"API_TOKEN missing from environment. Did you configure the .env file?"
|
||||
)
|
||||
if not FLUXIMMO_API_KEY:
|
||||
raise RuntimeError(
|
||||
"FLUXIMMO_API_KEY missing from environment. Did you configure the .env file?"
|
||||
)
|
||||
|
||||
app = Flask(__name__)
|
||||
CORS(app, resources={r"/*": {"origins": "*"}}, supports_credentials=True)
|
||||
@ -177,6 +190,15 @@ def _parse_int(value: Any, field_name: str) -> int:
|
||||
abort(400, description=f"Field '{field_name}' must be an integer")
|
||||
|
||||
|
||||
def _parse_optional_int(value: Any, field_name: str) -> int | None:
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
return _parse_int(value, field_name)
|
||||
except Exception:
|
||||
abort(400, description=f"Field '{field_name}' must be an integer or null")
|
||||
|
||||
|
||||
def _parse_uuid(value: Any, field_name: str) -> UUID:
|
||||
if isinstance(value, UUID):
|
||||
return value
|
||||
@ -188,6 +210,60 @@ def _parse_uuid(value: Any, field_name: str) -> UUID:
|
||||
abort(400, description=f"Field '{field_name}' must be a valid UUID")
|
||||
|
||||
|
||||
def _load_scraper_params(value: Any) -> Dict[str, Any]:
|
||||
if value is None:
|
||||
return {}
|
||||
if isinstance(value, Json):
|
||||
value = value.data
|
||||
if isinstance(value, dict):
|
||||
return deepcopy(value)
|
||||
if isinstance(value, str):
|
||||
try:
|
||||
parsed = json.loads(value)
|
||||
except json.JSONDecodeError:
|
||||
abort(400, description="Field 'params' must contain valid JSON")
|
||||
if isinstance(parsed, dict):
|
||||
return parsed
|
||||
abort(400, description="Field 'params' must decode to an object")
|
||||
abort(400, description="Field 'params' must be a JSON object or string")
|
||||
|
||||
|
||||
def build_scraper_params(
|
||||
params: Dict[str, Any],
|
||||
first_seen_days: int | None,
|
||||
last_seen_days: int | None,
|
||||
) -> Dict[str, Any]:
|
||||
now = datetime.now(timezone.utc)
|
||||
dynamic_params: Dict[str, Any] = {"meta": {}}
|
||||
|
||||
if first_seen_days is not None:
|
||||
dynamic_params["meta"]["firstSeenAt"] = {
|
||||
"min": (
|
||||
(now - timedelta(days=first_seen_days))
|
||||
.replace(microsecond=0)
|
||||
.isoformat()
|
||||
.replace("+00:00", "Z")
|
||||
)
|
||||
}
|
||||
|
||||
if last_seen_days is not None:
|
||||
dynamic_params["meta"]["lastSeenAt"] = {
|
||||
"max": (
|
||||
(now - timedelta(days=last_seen_days))
|
||||
.replace(microsecond=0)
|
||||
.isoformat()
|
||||
.replace("+00:00", "Z")
|
||||
)
|
||||
}
|
||||
|
||||
if not dynamic_params["meta"]:
|
||||
dynamic_params.pop("meta")
|
||||
|
||||
base = deepcopy(params)
|
||||
merged = always_merger.merge(base, dynamic_params)
|
||||
return merged
|
||||
|
||||
|
||||
def _require_bearer_token(header_value: str | None) -> str:
|
||||
if not header_value:
|
||||
abort(401, description="Missing bearer token")
|
||||
@ -426,10 +502,7 @@ def get_profile(profile_id: str):
|
||||
def create_profile():
|
||||
payload = _get_json_body()
|
||||
|
||||
profile_identifier = payload.get("profile_id")
|
||||
profile_uuid = (
|
||||
_parse_uuid(profile_identifier, "profile_id") if profile_identifier else uuid4()
|
||||
)
|
||||
profile_uuid = uuid4()
|
||||
|
||||
name = _parse_string(payload.get("name"), "name")
|
||||
description_value = payload.get("description")
|
||||
@ -666,7 +739,7 @@ def get_scraper(scraper_id: str):
|
||||
def create_scraper():
|
||||
payload = _get_json_body()
|
||||
|
||||
scraper_id = _parse_string(payload.get("id"), "id")
|
||||
scraper_id = str(uuid4())
|
||||
|
||||
data: Dict[str, Any] = {"id": scraper_id}
|
||||
|
||||
@ -735,5 +808,56 @@ def delete_scraper(scraper_id: str):
|
||||
return "", 204
|
||||
|
||||
|
||||
@app.post("/scrapers/count")
|
||||
def count_scraper_properties():
|
||||
payload = _get_json_body()
|
||||
|
||||
base_params = _load_scraper_params(payload.get("params"))
|
||||
first_seen_days = _parse_optional_int(
|
||||
payload.get("first_seen_days"), "first_seen_days"
|
||||
)
|
||||
last_seen_days = _parse_optional_int(
|
||||
payload.get("last_seen_days"), "last_seen_days"
|
||||
)
|
||||
|
||||
query_filters = build_scraper_params(base_params, first_seen_days, last_seen_days)
|
||||
flux_payload = {"query": query_filters}
|
||||
|
||||
headers = {
|
||||
"x-api-key": FLUXIMMO_API_KEY,
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
|
||||
try:
|
||||
response = requests.post(
|
||||
FLUXIMMO_COUNT_URL, json=flux_payload, headers=headers, timeout=15
|
||||
)
|
||||
except requests.RequestException:
|
||||
abort(502, description="Fluximmo request failed")
|
||||
|
||||
if response.status_code >= 400:
|
||||
try:
|
||||
detail = response.json()
|
||||
except ValueError:
|
||||
detail = response.text
|
||||
abort(502, description=f"Fluximmo error: {detail}")
|
||||
|
||||
try:
|
||||
response_data = response.json()
|
||||
except ValueError:
|
||||
abort(502, description="Fluximmo response was not JSON")
|
||||
|
||||
count = response_data.get("data", {}).get("count")
|
||||
if count is None:
|
||||
abort(502, description="Fluximmo response missing count")
|
||||
|
||||
try:
|
||||
count_value = int(count)
|
||||
except (TypeError, ValueError):
|
||||
abort(502, description="Fluximmo count is not an integer")
|
||||
|
||||
return jsonify({"count": count_value})
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
app.run(host="0.0.0.0", port=int(os.getenv("PORT", "8000")), debug=False)
|
||||
|
||||
@ -2,3 +2,5 @@ Flask>=3.0.0,<4.0.0
|
||||
python-dotenv>=1.0.0,<2.0.0
|
||||
psycopg[binary]>=3.1,<4.0
|
||||
Flask-Cors>=4.0.0,<5.0.0
|
||||
requests>=2.31.0,<3.0.0
|
||||
deepmerge>=1.1.0,<2.0.0
|
||||
|
||||
@ -5,6 +5,7 @@
|
||||
"name": "frontend",
|
||||
"dependencies": {
|
||||
"@radix-ui/react-checkbox": "^1.3.3",
|
||||
"@radix-ui/react-dialog": "^1.1.15",
|
||||
"@radix-ui/react-select": "^2.2.6",
|
||||
"@radix-ui/react-slot": "^1.2.3",
|
||||
"@radix-ui/react-tabs": "^1.1.13",
|
||||
@ -157,6 +158,8 @@
|
||||
|
||||
"@radix-ui/react-context": ["@radix-ui/react-context@1.1.2", "", { "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-jCi/QKUM2r1Ju5a3J64TH2A5SpKAgh0LpknyqdQ4m6DCV0xJ2HG1xARRwNGPQfi1SLdLWZ1OJz6F4OMBBNiGJA=="],
|
||||
|
||||
"@radix-ui/react-dialog": ["@radix-ui/react-dialog@1.1.15", "", { "dependencies": { "@radix-ui/primitive": "1.1.3", "@radix-ui/react-compose-refs": "1.1.2", "@radix-ui/react-context": "1.1.2", "@radix-ui/react-dismissable-layer": "1.1.11", "@radix-ui/react-focus-guards": "1.1.3", "@radix-ui/react-focus-scope": "1.1.7", "@radix-ui/react-id": "1.1.1", "@radix-ui/react-portal": "1.1.9", "@radix-ui/react-presence": "1.1.5", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-slot": "1.2.3", "@radix-ui/react-use-controllable-state": "1.2.2", "aria-hidden": "^1.2.4", "react-remove-scroll": "^2.6.3" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-TCglVRtzlffRNxRMEyR36DGBLJpeusFcgMVD9PZEzAKnUs1lKCgX5u9BmC2Yg+LL9MgZDugFFs1Vl+Jp4t/PGw=="],
|
||||
|
||||
"@radix-ui/react-direction": ["@radix-ui/react-direction@1.1.1", "", { "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-1UEWRX6jnOA2y4H5WczZ44gOOjTEmlqv1uNW4GAJEO5+bauCBhv8snY65Iw5/VOS/ghKN9gr2KjnLKxrsvoMVw=="],
|
||||
|
||||
"@radix-ui/react-dismissable-layer": ["@radix-ui/react-dismissable-layer@1.1.11", "", { "dependencies": { "@radix-ui/primitive": "1.1.3", "@radix-ui/react-compose-refs": "1.1.2", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-use-callback-ref": "1.1.1", "@radix-ui/react-use-escape-keydown": "1.1.1" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-Nqcp+t5cTB8BinFkZgXiMJniQH0PsUt2k51FUhbdfeKvc4ACcG2uQniY/8+h1Yv6Kza4Q7lD7PQV0z0oicE0Mg=="],
|
||||
|
||||
@ -11,6 +11,7 @@
|
||||
},
|
||||
"dependencies": {
|
||||
"@radix-ui/react-checkbox": "^1.3.3",
|
||||
"@radix-ui/react-dialog": "^1.1.15",
|
||||
"@radix-ui/react-select": "^2.2.6",
|
||||
"@radix-ui/react-slot": "^1.2.3",
|
||||
"@radix-ui/react-tabs": "^1.1.13",
|
||||
|
||||
141
frontend/src/components/ui/dialog.tsx
Normal file
141
frontend/src/components/ui/dialog.tsx
Normal file
@ -0,0 +1,141 @@
|
||||
import * as React from "react"
|
||||
import * as DialogPrimitive from "@radix-ui/react-dialog"
|
||||
import { XIcon } from "lucide-react"
|
||||
|
||||
import { cn } from "@/lib/utils"
|
||||
|
||||
function Dialog({
|
||||
...props
|
||||
}: React.ComponentProps<typeof DialogPrimitive.Root>) {
|
||||
return <DialogPrimitive.Root data-slot="dialog" {...props} />
|
||||
}
|
||||
|
||||
function DialogTrigger({
|
||||
...props
|
||||
}: React.ComponentProps<typeof DialogPrimitive.Trigger>) {
|
||||
return <DialogPrimitive.Trigger data-slot="dialog-trigger" {...props} />
|
||||
}
|
||||
|
||||
function DialogPortal({
|
||||
...props
|
||||
}: React.ComponentProps<typeof DialogPrimitive.Portal>) {
|
||||
return <DialogPrimitive.Portal data-slot="dialog-portal" {...props} />
|
||||
}
|
||||
|
||||
function DialogClose({
|
||||
...props
|
||||
}: React.ComponentProps<typeof DialogPrimitive.Close>) {
|
||||
return <DialogPrimitive.Close data-slot="dialog-close" {...props} />
|
||||
}
|
||||
|
||||
function DialogOverlay({
|
||||
className,
|
||||
...props
|
||||
}: React.ComponentProps<typeof DialogPrimitive.Overlay>) {
|
||||
return (
|
||||
<DialogPrimitive.Overlay
|
||||
data-slot="dialog-overlay"
|
||||
className={cn(
|
||||
"data-[state=open]:animate-in data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=open]:fade-in-0 fixed inset-0 z-50 bg-black/50",
|
||||
className
|
||||
)}
|
||||
{...props}
|
||||
/>
|
||||
)
|
||||
}
|
||||
|
||||
function DialogContent({
|
||||
className,
|
||||
children,
|
||||
showCloseButton = true,
|
||||
...props
|
||||
}: React.ComponentProps<typeof DialogPrimitive.Content> & {
|
||||
showCloseButton?: boolean
|
||||
}) {
|
||||
return (
|
||||
<DialogPortal data-slot="dialog-portal">
|
||||
<DialogOverlay />
|
||||
<DialogPrimitive.Content
|
||||
data-slot="dialog-content"
|
||||
className={cn(
|
||||
"bg-background data-[state=open]:animate-in data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=open]:fade-in-0 data-[state=closed]:zoom-out-95 data-[state=open]:zoom-in-95 fixed top-[50%] left-[50%] z-50 grid w-full max-w-[calc(100%-2rem)] translate-x-[-50%] translate-y-[-50%] gap-4 rounded-lg border p-6 shadow-lg duration-200 sm:max-w-lg",
|
||||
className
|
||||
)}
|
||||
{...props}
|
||||
>
|
||||
{children}
|
||||
{showCloseButton && (
|
||||
<DialogPrimitive.Close
|
||||
data-slot="dialog-close"
|
||||
className="ring-offset-background focus:ring-ring data-[state=open]:bg-accent data-[state=open]:text-muted-foreground absolute top-4 right-4 rounded-xs opacity-70 transition-opacity hover:opacity-100 focus:ring-2 focus:ring-offset-2 focus:outline-hidden disabled:pointer-events-none [&_svg]:pointer-events-none [&_svg]:shrink-0 [&_svg:not([class*='size-'])]:size-4"
|
||||
>
|
||||
<XIcon />
|
||||
<span className="sr-only">Close</span>
|
||||
</DialogPrimitive.Close>
|
||||
)}
|
||||
</DialogPrimitive.Content>
|
||||
</DialogPortal>
|
||||
)
|
||||
}
|
||||
|
||||
function DialogHeader({ className, ...props }: React.ComponentProps<"div">) {
|
||||
return (
|
||||
<div
|
||||
data-slot="dialog-header"
|
||||
className={cn("flex flex-col gap-2 text-center sm:text-left", className)}
|
||||
{...props}
|
||||
/>
|
||||
)
|
||||
}
|
||||
|
||||
function DialogFooter({ className, ...props }: React.ComponentProps<"div">) {
|
||||
return (
|
||||
<div
|
||||
data-slot="dialog-footer"
|
||||
className={cn(
|
||||
"flex flex-col-reverse gap-2 sm:flex-row sm:justify-end",
|
||||
className
|
||||
)}
|
||||
{...props}
|
||||
/>
|
||||
)
|
||||
}
|
||||
|
||||
function DialogTitle({
|
||||
className,
|
||||
...props
|
||||
}: React.ComponentProps<typeof DialogPrimitive.Title>) {
|
||||
return (
|
||||
<DialogPrimitive.Title
|
||||
data-slot="dialog-title"
|
||||
className={cn("text-lg leading-none font-semibold", className)}
|
||||
{...props}
|
||||
/>
|
||||
)
|
||||
}
|
||||
|
||||
function DialogDescription({
|
||||
className,
|
||||
...props
|
||||
}: React.ComponentProps<typeof DialogPrimitive.Description>) {
|
||||
return (
|
||||
<DialogPrimitive.Description
|
||||
data-slot="dialog-description"
|
||||
className={cn("text-muted-foreground text-sm", className)}
|
||||
{...props}
|
||||
/>
|
||||
)
|
||||
}
|
||||
|
||||
export {
|
||||
Dialog,
|
||||
DialogClose,
|
||||
DialogContent,
|
||||
DialogDescription,
|
||||
DialogFooter,
|
||||
DialogHeader,
|
||||
DialogOverlay,
|
||||
DialogPortal,
|
||||
DialogTitle,
|
||||
DialogTrigger,
|
||||
}
|
||||
@ -30,7 +30,6 @@ export interface Profile {
|
||||
}
|
||||
|
||||
interface CreateProfilePayload {
|
||||
profile_id?: string;
|
||||
name: string;
|
||||
description?: string | null;
|
||||
criteria: JsonObject;
|
||||
@ -39,7 +38,6 @@ interface CreateProfilePayload {
|
||||
}
|
||||
|
||||
interface ProfileFormValues {
|
||||
profile_id: string;
|
||||
name: string;
|
||||
description: string;
|
||||
criteria: JsonObject;
|
||||
@ -78,12 +76,6 @@ function transformPayload(values: ProfileFormValues): CreateProfilePayload {
|
||||
is_active: values.is_active,
|
||||
};
|
||||
|
||||
const profileId = values.profile_id.trim();
|
||||
|
||||
if (profileId) {
|
||||
payload.profile_id = profileId;
|
||||
}
|
||||
|
||||
const description = values.description.trim();
|
||||
|
||||
if (description) {
|
||||
@ -109,7 +101,6 @@ const profileSchema = getSchema("profile");
|
||||
|
||||
function createDefaultValues(): ProfileFormValues {
|
||||
return {
|
||||
profile_id: "",
|
||||
name: "",
|
||||
description: "",
|
||||
criteria: buildDefaultValue(profileSchema) as JsonObject,
|
||||
@ -228,12 +219,7 @@ export function ProfilesTab() {
|
||||
const updateProfileMutation = useMutation({
|
||||
mutationFn: ({ id, data }: { id: string; data: ProfileFormValues }) => {
|
||||
const payload = transformPayload(data);
|
||||
const { profile_id: _ignored, ...body } = payload;
|
||||
|
||||
return api.put<Omit<CreateProfilePayload, "profile_id">, Profile>(
|
||||
`/profiles/${id}`,
|
||||
body
|
||||
);
|
||||
return api.put<CreateProfilePayload, Profile>(`/profiles/${id}`, payload);
|
||||
},
|
||||
onSuccess: () => {
|
||||
queryClient.invalidateQueries({ queryKey: ["profiles"] });
|
||||
@ -285,18 +271,6 @@ export function ProfilesTab() {
|
||||
<CardContent>
|
||||
<form className="space-y-6" onSubmit={onSubmit}>
|
||||
<div className="grid gap-4 md:grid-cols-2">
|
||||
<div className="space-y-1">
|
||||
<label className="text-sm font-medium" htmlFor="profile_id">
|
||||
Identifiant (optionnel)
|
||||
</label>
|
||||
<Input
|
||||
id="profile_id"
|
||||
placeholder="UUID personnalise"
|
||||
disabled={Boolean(editingProfileId)}
|
||||
{...form.register("profile_id")}
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div className="space-y-1">
|
||||
<label className="text-sm font-medium" htmlFor="name">
|
||||
Nom
|
||||
@ -444,7 +418,6 @@ export function ProfilesTab() {
|
||||
<table className="min-w-full table-fixed border-collapse text-sm">
|
||||
<thead>
|
||||
<tr className="text-left">
|
||||
<th className="border-b px-3 py-2 font-semibold">ID</th>
|
||||
<th className="border-b px-3 py-2 font-semibold">Nom</th>
|
||||
<th className="border-b px-3 py-2 font-semibold">Description</th>
|
||||
<th className="border-b px-3 py-2 font-semibold">Criteres</th>
|
||||
@ -456,10 +429,7 @@ export function ProfilesTab() {
|
||||
<tbody>
|
||||
{profiles.map((profile) => (
|
||||
<tr key={profile.profile_id} className="align-top">
|
||||
<td className="border-b px-3 py-2 font-mono text-xs">
|
||||
{profile.profile_id}
|
||||
</td>
|
||||
<td className="border-b px-3 py-2">{profile.name}</td>
|
||||
<td className="border-b px-3 py-2 font-medium">{profile.name}</td>
|
||||
<td className="border-b px-3 py-2 text-muted-foreground">
|
||||
{profile.description ?? "-"}
|
||||
</td>
|
||||
@ -496,20 +466,19 @@ export function ProfilesTab() {
|
||||
const createdAtDate = profile.created_at
|
||||
? new Date(profile.created_at)
|
||||
: null;
|
||||
const createdAtInput =
|
||||
createdAtDate && !Number.isNaN(createdAtDate.getTime())
|
||||
? createdAtDate.toISOString().slice(0, 16)
|
||||
: "";
|
||||
form.reset({
|
||||
profile_id: profile.profile_id,
|
||||
name: profile.name,
|
||||
description: profile.description ?? "",
|
||||
criteria: normalizeProfileCriteriaForForm(
|
||||
profile.criteria
|
||||
),
|
||||
created_at: createdAtInput,
|
||||
is_active: profile.is_active,
|
||||
});
|
||||
const createdAtInput =
|
||||
createdAtDate && !Number.isNaN(createdAtDate.getTime())
|
||||
? createdAtDate.toISOString().slice(0, 16)
|
||||
: "";
|
||||
form.reset({
|
||||
name: profile.name,
|
||||
description: profile.description ?? "",
|
||||
criteria: normalizeProfileCriteriaForForm(
|
||||
profile.criteria
|
||||
),
|
||||
created_at: createdAtInput,
|
||||
is_active: profile.is_active,
|
||||
});
|
||||
}}
|
||||
>
|
||||
Modifier
|
||||
|
||||
@ -16,6 +16,15 @@ import {
|
||||
import { Checkbox } from "@/components/ui/checkbox";
|
||||
import { Input } from "@/components/ui/input";
|
||||
import { SchemaAwareJsonField } from "@/components/schema-builder";
|
||||
import {
|
||||
Dialog,
|
||||
DialogClose,
|
||||
DialogContent,
|
||||
DialogDescription,
|
||||
DialogFooter,
|
||||
DialogHeader,
|
||||
DialogTitle,
|
||||
} from "@/components/ui/dialog";
|
||||
import { getSchema } from "@/schemas/loader";
|
||||
import { buildDefaultValue } from "@/schemas/utils";
|
||||
|
||||
@ -35,7 +44,6 @@ export interface Scraper {
|
||||
}
|
||||
|
||||
interface ScraperFormValues {
|
||||
id: string;
|
||||
params: JsonObject;
|
||||
frequency: string;
|
||||
task_name: string;
|
||||
@ -50,7 +58,7 @@ interface ScraperFormValues {
|
||||
}
|
||||
|
||||
interface CreateScraperPayload {
|
||||
id: string;
|
||||
id?: string;
|
||||
params?: string | null;
|
||||
frequency?: string | null;
|
||||
task_name?: string | null;
|
||||
@ -105,9 +113,7 @@ function parseInteger(value: string) {
|
||||
}
|
||||
|
||||
function transformPayload(values: ScraperFormValues): CreateScraperPayload {
|
||||
const payload: CreateScraperPayload = {
|
||||
id: values.id.trim(),
|
||||
};
|
||||
const payload: CreateScraperPayload = {};
|
||||
|
||||
const paramsObject = values.params ?? {};
|
||||
|
||||
@ -159,7 +165,6 @@ function transformPayload(values: ScraperFormValues): CreateScraperPayload {
|
||||
|
||||
function createDefaultScraperValues(): ScraperFormValues {
|
||||
return {
|
||||
id: "",
|
||||
params: buildDefaultValue(scraperSchema) as JsonObject,
|
||||
frequency: "",
|
||||
task_name: "",
|
||||
@ -190,6 +195,7 @@ export function ScrapersTab() {
|
||||
const [paramsValid, setParamsValid] = useState(true);
|
||||
const [editingScraperId, setEditingScraperId] = useState<string | null>(null);
|
||||
const [statusMessage, setStatusMessage] = useState<string | null>(null);
|
||||
const [isCountDialogOpen, setIsCountDialogOpen] = useState(false);
|
||||
const queryClient = useQueryClient();
|
||||
const initialValues = useMemo(() => createDefaultScraperValues(), []);
|
||||
const form = useForm<ScraperFormValues>({
|
||||
@ -217,9 +223,7 @@ export function ScrapersTab() {
|
||||
const updateScraperMutation = useMutation({
|
||||
mutationFn: ({ id, data }: { id: string; data: ScraperFormValues }) => {
|
||||
const payload = transformPayload(data);
|
||||
const { id: _ignored, ...body } = payload;
|
||||
|
||||
return api.put<Omit<CreateScraperPayload, "id">, Scraper>(`/scrapers/${id}`, body);
|
||||
return api.put<CreateScraperPayload, Scraper>(`/scrapers/${id}`, payload);
|
||||
},
|
||||
onSuccess: () => {
|
||||
queryClient.invalidateQueries({ queryKey: ["scrapers"] });
|
||||
@ -243,6 +247,16 @@ export function ScrapersTab() {
|
||||
},
|
||||
});
|
||||
|
||||
type ScraperCountResponse = { count: number };
|
||||
|
||||
const countScraperMutation = useMutation({
|
||||
mutationFn: (values: ScraperFormValues) =>
|
||||
api.post<CreateScraperPayload, ScraperCountResponse>(
|
||||
"/scrapers/count",
|
||||
transformPayload(values)
|
||||
),
|
||||
});
|
||||
|
||||
const onSubmit = form.handleSubmit(async (values) => {
|
||||
setStatusMessage(null);
|
||||
if (editingScraperId) {
|
||||
@ -257,6 +271,19 @@ export function ScrapersTab() {
|
||||
(updateScraperMutation.isError && getErrorMessage(updateScraperMutation.error)) ||
|
||||
null;
|
||||
|
||||
const handleOpenCountDialog = async () => {
|
||||
setStatusMessage(null);
|
||||
const isFormValid = await form.trigger();
|
||||
|
||||
if (!isFormValid || !paramsValid) {
|
||||
return;
|
||||
}
|
||||
|
||||
countScraperMutation.reset();
|
||||
setIsCountDialogOpen(true);
|
||||
countScraperMutation.mutate(form.getValues());
|
||||
};
|
||||
|
||||
const scrapers = useMemo(() => scrapersQuery.data ?? [], [scrapersQuery.data]);
|
||||
|
||||
return (
|
||||
@ -265,28 +292,21 @@ export function ScrapersTab() {
|
||||
<CardHeader>
|
||||
<CardTitle>Creer un scraper</CardTitle>
|
||||
<CardDescription>
|
||||
Definissez l'identifiant du scraper, la configuration et les parametres d'execution.
|
||||
Definissez la configuration et les parametres d'execution du scraper.
|
||||
</CardDescription>
|
||||
</CardHeader>
|
||||
<CardContent>
|
||||
<form className="space-y-6" onSubmit={onSubmit}>
|
||||
<div className="grid gap-4 md:grid-cols-2">
|
||||
<div className="space-y-1 md:col-span-2">
|
||||
<label className="text-sm font-medium" htmlFor="scraper_id">
|
||||
Identifiant
|
||||
<label className="text-sm font-medium" htmlFor="task_name">
|
||||
Nom du scraper
|
||||
</label>
|
||||
<Input
|
||||
id="scraper_id"
|
||||
placeholder="ex: scraper_immobilier"
|
||||
aria-invalid={Boolean(form.formState.errors.id)}
|
||||
disabled={Boolean(editingScraperId)}
|
||||
{...form.register("id", { required: "L'identifiant est obligatoire." })}
|
||||
id="task_name"
|
||||
placeholder="Nom interne"
|
||||
{...form.register("task_name")}
|
||||
/>
|
||||
{form.formState.errors.id ? (
|
||||
<p className="text-destructive text-xs">
|
||||
{form.formState.errors.id.message}
|
||||
</p>
|
||||
) : null}
|
||||
</div>
|
||||
|
||||
<div className="space-y-1">
|
||||
@ -300,17 +320,6 @@ export function ScrapersTab() {
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div className="space-y-1">
|
||||
<label className="text-sm font-medium" htmlFor="task_name">
|
||||
Nom de tache
|
||||
</label>
|
||||
<Input
|
||||
id="task_name"
|
||||
placeholder="Nom interne"
|
||||
{...form.register("task_name")}
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div className="space-y-1">
|
||||
<label className="text-sm font-medium" htmlFor="property_types">
|
||||
Types de biens
|
||||
@ -438,6 +447,15 @@ export function ScrapersTab() {
|
||||
<p className="text-destructive text-sm">Corrigez les erreurs du schema avant l'envoi.</p>
|
||||
) : null}
|
||||
|
||||
<Button
|
||||
type="button"
|
||||
variant="secondary"
|
||||
disabled={countScraperMutation.isPending}
|
||||
onClick={handleOpenCountDialog}
|
||||
>
|
||||
{countScraperMutation.isPending ? "Calcul..." : "Compter les resultats"}
|
||||
</Button>
|
||||
|
||||
{editingScraperId ? (
|
||||
<Button
|
||||
type="button"
|
||||
@ -477,9 +495,8 @@ export function ScrapersTab() {
|
||||
<table className="min-w-full table-fixed border-collapse text-sm">
|
||||
<thead>
|
||||
<tr className="text-left">
|
||||
<th className="border-b px-3 py-2 font-semibold">ID</th>
|
||||
<th className="border-b px-3 py-2 font-semibold">Nom</th>
|
||||
<th className="border-b px-3 py-2 font-semibold">Frequence</th>
|
||||
<th className="border-b px-3 py-2 font-semibold">Tache</th>
|
||||
<th className="border-b px-3 py-2 font-semibold">Parametres</th>
|
||||
<th className="border-b px-3 py-2 font-semibold">Etat</th>
|
||||
<th className="border-b px-3 py-2 font-semibold">Limites</th>
|
||||
@ -492,11 +509,10 @@ export function ScrapersTab() {
|
||||
|
||||
return (
|
||||
<tr key={scraper.id} className="align-top">
|
||||
<td className="border-b px-3 py-2 font-mono text-xs">{scraper.id}</td>
|
||||
<td className="border-b px-3 py-2 font-medium">{scraper.task_name ?? "-"}</td>
|
||||
<td className="border-b px-3 py-2 text-muted-foreground">
|
||||
{scraper.frequency ?? "-"}
|
||||
</td>
|
||||
<td className="border-b px-3 py-2">{scraper.task_name ?? "-"}</td>
|
||||
<td className="border-b px-3 py-2">
|
||||
{paramsDisplay ? (
|
||||
<pre className="max-h-40 overflow-auto rounded bg-muted/50 p-2 text-xs">
|
||||
@ -554,7 +570,6 @@ export function ScrapersTab() {
|
||||
}
|
||||
}
|
||||
form.reset({
|
||||
id: scraper.id,
|
||||
params: parsedParams,
|
||||
frequency: scraper.frequency ?? "",
|
||||
task_name: scraper.task_name ?? "",
|
||||
@ -598,6 +613,49 @@ export function ScrapersTab() {
|
||||
)}
|
||||
</CardContent>
|
||||
</Card>
|
||||
|
||||
<Dialog
|
||||
open={isCountDialogOpen}
|
||||
onOpenChange={(open) => {
|
||||
setIsCountDialogOpen(open);
|
||||
if (!open) {
|
||||
countScraperMutation.reset();
|
||||
}
|
||||
}}
|
||||
>
|
||||
<DialogContent>
|
||||
<DialogHeader>
|
||||
<DialogTitle>Estimation du nombre d'annonces</DialogTitle>
|
||||
<DialogDescription>
|
||||
Calcul base sur la configuration actuelle du scraper.
|
||||
</DialogDescription>
|
||||
</DialogHeader>
|
||||
<div className="min-h-[80px]">
|
||||
{countScraperMutation.isPending ? (
|
||||
<p className="text-sm text-muted-foreground">Calcul en cours...</p>
|
||||
) : countScraperMutation.isError ? (
|
||||
<p className="text-destructive text-sm">
|
||||
{getErrorMessage(countScraperMutation.error)}
|
||||
</p>
|
||||
) : countScraperMutation.data ? (
|
||||
<p className="text-sm">
|
||||
Ce scraper correspond a
|
||||
<span className="font-semibold"> {countScraperMutation.data.count} </span>
|
||||
annonce(s) potentielle(s).
|
||||
</p>
|
||||
) : (
|
||||
<p className="text-muted-foreground text-sm">
|
||||
Aucun calcul n'a encore ete effectue.
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
<DialogFooter>
|
||||
<DialogClose asChild>
|
||||
<Button variant="outline">Fermer</Button>
|
||||
</DialogClose>
|
||||
</DialogFooter>
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user