diff --git a/backend/api_summary.json b/backend/api_summary.json index f82338d..e0d2d48 100644 --- a/backend/api_summary.json +++ b/backend/api_summary.json @@ -10,6 +10,11 @@ "allow_origin": "*", "allow_credentials": true, "notes": "Server responds to OPTIONS preflight without authentication and accepts credentialed requests from any origin." + }, + "environment": { + "API_TOKEN": "Bearer token expected on every request.", + "FLUXIMMO_API_KEY": "Credential forwarded to Fluximmo analytics count endpoint.", + "FLUXIMMO_COUNT_URL": "Optional override for Fluximmo count API base URL (defaults to https://analytics.fluximmo.io/properties/count)." } }, "authentication": { @@ -37,7 +42,7 @@ "Profile": { "description": "Investment profile descriptor persisted in users_investmentprofile.", "fields": [ - {"name": "profile_id", "type": "uuid", "nullable": false, "read_only": false, "description": "Primary key. Optional on create; autogenerated when omitted."}, + {"name": "profile_id", "type": "uuid", "nullable": false, "read_only": true, "description": "Primary key generated server-side."}, {"name": "name", "type": "string", "nullable": false, "read_only": false, "description": "Human-readable profile label."}, {"name": "description", "type": "string", "nullable": true, "read_only": false, "description": "Optional free-form description. Empty string preserved."}, {"name": "criteria", "type": "json", "nullable": false, "read_only": false, "description": "Arbitrary JSON object/array describing selection rules."}, @@ -64,7 +69,7 @@ "Scraper": { "description": "Scraper configuration persisted in scraper table.", "fields": [ - {"name": "id", "type": "string", "nullable": false, "read_only": false, "description": "Primary key identifier."}, + {"name": "id", "type": "uuid", "nullable": false, "read_only": true, "description": "Primary key generated server-side."}, {"name": "params", "type": "string", "nullable": true, "read_only": false, "description": "Serialized scraper parameters."}, {"name": "last_seen_days", "type": "integer", "nullable": true, "read_only": false, "description": "Number of days since listings were last seen."}, {"name": "first_seen_days", "type": "integer", "nullable": true, "read_only": false, "description": "Number of days since listings were first seen."}, @@ -133,7 +138,6 @@ "type": "object", "required_fields": ["name", "criteria"], "fields": { - "profile_id": {"type": "uuid", "nullable": false, "description": "Optional explicit UUID. Auto-generated when omitted."}, "name": {"type": "string", "nullable": false, "description": "Profile name."}, "description": {"type": "string", "nullable": true, "description": "Optional description. Use null to clear."}, "criteria": {"type": "json", "nullable": false, "description": "Required JSON criteria payload."}, @@ -394,9 +398,8 @@ }, "body": { "type": "object", - "required_fields": ["id"], + "required_fields": [], "fields": { - "id": {"type": "string", "nullable": false, "description": "Scraper primary key."}, "params": {"type": "string", "nullable": true, "description": "Serialized parameter payload."}, "frequency": {"type": "string", "nullable": true, "description": "Execution cadence descriptor."}, "task_name": {"type": "string", "nullable": true, "description": "Linked task name."}, @@ -481,6 +484,34 @@ "404": {"description": "Scraper not found."}, "503": {"description": "Database unavailable."} } + }, + { + "operation_id": "countScraperProperties", + "method": "POST", + "path": "/scrapers/count", + "authenticated": true, + "description": "Return the Fluximmo property count derived from a scraper configuration.", + "request": { + "headers": { + "Content-Type": {"type": "string", "required": true, "example": "application/json"}, + "Authorization": {"type": "string", "required": true, "description": "Bearer "} + }, + "body": { + "type": "object", + "required_fields": ["params"], + "fields": { + "params": {"type": "object", "nullable": false, "description": "Base Fluximmo search filters. Accepts a JSON object or string-encoded JSON."}, + "first_seen_days": {"type": "integer", "nullable": true, "description": "Days before now for firstSeenAt.min."}, + "last_seen_days": {"type": "integer", "nullable": true, "description": "Days before now for lastSeenAt.max."} + } + } + }, + "responses": { + "200": {"description": "Count retrieved.", "body": {"type": "object", "fields": {"count": {"type": "integer", "nullable": false}}}}, + "400": {"description": "Validation failure."}, + "401": {"description": "Missing or invalid bearer token."}, + "502": {"description": "Fluximmo upstream error."} + } } ] } diff --git a/backend/app.py b/backend/app.py index 21007d2..136d3aa 100644 --- a/backend/app.py +++ b/backend/app.py @@ -3,7 +3,9 @@ from __future__ import annotations import os -from datetime import datetime, timezone +import json +from copy import deepcopy +from datetime import datetime, timedelta, timezone from typing import Any, Dict, Iterable, List, Mapping, MutableMapping from uuid import UUID, uuid4 @@ -11,13 +13,20 @@ import psycopg from psycopg import sql from psycopg.rows import dict_row from psycopg.types.json import Json +from deepmerge import always_merger from dotenv import load_dotenv from flask import Flask, abort, jsonify, request, g from flask_cors import CORS +import requests load_dotenv() API_TOKEN = os.getenv("API_TOKEN") +FLUXIMMO_API_KEY = os.getenv("FLUXIMMO_API_KEY") +FLUXIMMO_COUNT_URL = os.getenv( + "FLUXIMMO_COUNT_URL", + "https://analytics.fluximmo.io/properties/count", +) REQUIRED_DB_SETTINGS = { @@ -57,6 +66,10 @@ if not API_TOKEN: raise RuntimeError( "API_TOKEN missing from environment. Did you configure the .env file?" ) +if not FLUXIMMO_API_KEY: + raise RuntimeError( + "FLUXIMMO_API_KEY missing from environment. Did you configure the .env file?" + ) app = Flask(__name__) CORS(app, resources={r"/*": {"origins": "*"}}, supports_credentials=True) @@ -177,6 +190,15 @@ def _parse_int(value: Any, field_name: str) -> int: abort(400, description=f"Field '{field_name}' must be an integer") +def _parse_optional_int(value: Any, field_name: str) -> int | None: + if value is None: + return None + try: + return _parse_int(value, field_name) + except Exception: + abort(400, description=f"Field '{field_name}' must be an integer or null") + + def _parse_uuid(value: Any, field_name: str) -> UUID: if isinstance(value, UUID): return value @@ -188,6 +210,60 @@ def _parse_uuid(value: Any, field_name: str) -> UUID: abort(400, description=f"Field '{field_name}' must be a valid UUID") +def _load_scraper_params(value: Any) -> Dict[str, Any]: + if value is None: + return {} + if isinstance(value, Json): + value = value.data + if isinstance(value, dict): + return deepcopy(value) + if isinstance(value, str): + try: + parsed = json.loads(value) + except json.JSONDecodeError: + abort(400, description="Field 'params' must contain valid JSON") + if isinstance(parsed, dict): + return parsed + abort(400, description="Field 'params' must decode to an object") + abort(400, description="Field 'params' must be a JSON object or string") + + +def build_scraper_params( + params: Dict[str, Any], + first_seen_days: int | None, + last_seen_days: int | None, +) -> Dict[str, Any]: + now = datetime.now(timezone.utc) + dynamic_params: Dict[str, Any] = {"meta": {}} + + if first_seen_days is not None: + dynamic_params["meta"]["firstSeenAt"] = { + "min": ( + (now - timedelta(days=first_seen_days)) + .replace(microsecond=0) + .isoformat() + .replace("+00:00", "Z") + ) + } + + if last_seen_days is not None: + dynamic_params["meta"]["lastSeenAt"] = { + "max": ( + (now - timedelta(days=last_seen_days)) + .replace(microsecond=0) + .isoformat() + .replace("+00:00", "Z") + ) + } + + if not dynamic_params["meta"]: + dynamic_params.pop("meta") + + base = deepcopy(params) + merged = always_merger.merge(base, dynamic_params) + return merged + + def _require_bearer_token(header_value: str | None) -> str: if not header_value: abort(401, description="Missing bearer token") @@ -426,10 +502,7 @@ def get_profile(profile_id: str): def create_profile(): payload = _get_json_body() - profile_identifier = payload.get("profile_id") - profile_uuid = ( - _parse_uuid(profile_identifier, "profile_id") if profile_identifier else uuid4() - ) + profile_uuid = uuid4() name = _parse_string(payload.get("name"), "name") description_value = payload.get("description") @@ -666,7 +739,7 @@ def get_scraper(scraper_id: str): def create_scraper(): payload = _get_json_body() - scraper_id = _parse_string(payload.get("id"), "id") + scraper_id = str(uuid4()) data: Dict[str, Any] = {"id": scraper_id} @@ -735,5 +808,56 @@ def delete_scraper(scraper_id: str): return "", 204 +@app.post("/scrapers/count") +def count_scraper_properties(): + payload = _get_json_body() + + base_params = _load_scraper_params(payload.get("params")) + first_seen_days = _parse_optional_int( + payload.get("first_seen_days"), "first_seen_days" + ) + last_seen_days = _parse_optional_int( + payload.get("last_seen_days"), "last_seen_days" + ) + + query_filters = build_scraper_params(base_params, first_seen_days, last_seen_days) + flux_payload = {"query": query_filters} + + headers = { + "x-api-key": FLUXIMMO_API_KEY, + "Content-Type": "application/json", + } + + try: + response = requests.post( + FLUXIMMO_COUNT_URL, json=flux_payload, headers=headers, timeout=15 + ) + except requests.RequestException: + abort(502, description="Fluximmo request failed") + + if response.status_code >= 400: + try: + detail = response.json() + except ValueError: + detail = response.text + abort(502, description=f"Fluximmo error: {detail}") + + try: + response_data = response.json() + except ValueError: + abort(502, description="Fluximmo response was not JSON") + + count = response_data.get("data", {}).get("count") + if count is None: + abort(502, description="Fluximmo response missing count") + + try: + count_value = int(count) + except (TypeError, ValueError): + abort(502, description="Fluximmo count is not an integer") + + return jsonify({"count": count_value}) + + if __name__ == "__main__": app.run(host="0.0.0.0", port=int(os.getenv("PORT", "8000")), debug=False) diff --git a/backend/requirements.txt b/backend/requirements.txt index 49dcb5f..e1fc536 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -2,3 +2,5 @@ Flask>=3.0.0,<4.0.0 python-dotenv>=1.0.0,<2.0.0 psycopg[binary]>=3.1,<4.0 Flask-Cors>=4.0.0,<5.0.0 +requests>=2.31.0,<3.0.0 +deepmerge>=1.1.0,<2.0.0 diff --git a/frontend/bun.lock b/frontend/bun.lock index 0f06ea6..e4c9bfa 100644 --- a/frontend/bun.lock +++ b/frontend/bun.lock @@ -5,6 +5,7 @@ "name": "frontend", "dependencies": { "@radix-ui/react-checkbox": "^1.3.3", + "@radix-ui/react-dialog": "^1.1.15", "@radix-ui/react-select": "^2.2.6", "@radix-ui/react-slot": "^1.2.3", "@radix-ui/react-tabs": "^1.1.13", @@ -157,6 +158,8 @@ "@radix-ui/react-context": ["@radix-ui/react-context@1.1.2", "", { "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-jCi/QKUM2r1Ju5a3J64TH2A5SpKAgh0LpknyqdQ4m6DCV0xJ2HG1xARRwNGPQfi1SLdLWZ1OJz6F4OMBBNiGJA=="], + "@radix-ui/react-dialog": ["@radix-ui/react-dialog@1.1.15", "", { "dependencies": { "@radix-ui/primitive": "1.1.3", "@radix-ui/react-compose-refs": "1.1.2", "@radix-ui/react-context": "1.1.2", "@radix-ui/react-dismissable-layer": "1.1.11", "@radix-ui/react-focus-guards": "1.1.3", "@radix-ui/react-focus-scope": "1.1.7", "@radix-ui/react-id": "1.1.1", "@radix-ui/react-portal": "1.1.9", "@radix-ui/react-presence": "1.1.5", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-slot": "1.2.3", "@radix-ui/react-use-controllable-state": "1.2.2", "aria-hidden": "^1.2.4", "react-remove-scroll": "^2.6.3" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-TCglVRtzlffRNxRMEyR36DGBLJpeusFcgMVD9PZEzAKnUs1lKCgX5u9BmC2Yg+LL9MgZDugFFs1Vl+Jp4t/PGw=="], + "@radix-ui/react-direction": ["@radix-ui/react-direction@1.1.1", "", { "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-1UEWRX6jnOA2y4H5WczZ44gOOjTEmlqv1uNW4GAJEO5+bauCBhv8snY65Iw5/VOS/ghKN9gr2KjnLKxrsvoMVw=="], "@radix-ui/react-dismissable-layer": ["@radix-ui/react-dismissable-layer@1.1.11", "", { "dependencies": { "@radix-ui/primitive": "1.1.3", "@radix-ui/react-compose-refs": "1.1.2", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-use-callback-ref": "1.1.1", "@radix-ui/react-use-escape-keydown": "1.1.1" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-Nqcp+t5cTB8BinFkZgXiMJniQH0PsUt2k51FUhbdfeKvc4ACcG2uQniY/8+h1Yv6Kza4Q7lD7PQV0z0oicE0Mg=="], diff --git a/frontend/package.json b/frontend/package.json index 0efe3f6..c76c0ae 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -11,6 +11,7 @@ }, "dependencies": { "@radix-ui/react-checkbox": "^1.3.3", + "@radix-ui/react-dialog": "^1.1.15", "@radix-ui/react-select": "^2.2.6", "@radix-ui/react-slot": "^1.2.3", "@radix-ui/react-tabs": "^1.1.13", diff --git a/frontend/src/components/ui/dialog.tsx b/frontend/src/components/ui/dialog.tsx new file mode 100644 index 0000000..6cb123b --- /dev/null +++ b/frontend/src/components/ui/dialog.tsx @@ -0,0 +1,141 @@ +import * as React from "react" +import * as DialogPrimitive from "@radix-ui/react-dialog" +import { XIcon } from "lucide-react" + +import { cn } from "@/lib/utils" + +function Dialog({ + ...props +}: React.ComponentProps) { + return +} + +function DialogTrigger({ + ...props +}: React.ComponentProps) { + return +} + +function DialogPortal({ + ...props +}: React.ComponentProps) { + return +} + +function DialogClose({ + ...props +}: React.ComponentProps) { + return +} + +function DialogOverlay({ + className, + ...props +}: React.ComponentProps) { + return ( + + ) +} + +function DialogContent({ + className, + children, + showCloseButton = true, + ...props +}: React.ComponentProps & { + showCloseButton?: boolean +}) { + return ( + + + + {children} + {showCloseButton && ( + + + Close + + )} + + + ) +} + +function DialogHeader({ className, ...props }: React.ComponentProps<"div">) { + return ( +
+ ) +} + +function DialogFooter({ className, ...props }: React.ComponentProps<"div">) { + return ( +
+ ) +} + +function DialogTitle({ + className, + ...props +}: React.ComponentProps) { + return ( + + ) +} + +function DialogDescription({ + className, + ...props +}: React.ComponentProps) { + return ( + + ) +} + +export { + Dialog, + DialogClose, + DialogContent, + DialogDescription, + DialogFooter, + DialogHeader, + DialogOverlay, + DialogPortal, + DialogTitle, + DialogTrigger, +} diff --git a/frontend/src/features/profiles/ProfilesTab.tsx b/frontend/src/features/profiles/ProfilesTab.tsx index c6de598..5e73120 100644 --- a/frontend/src/features/profiles/ProfilesTab.tsx +++ b/frontend/src/features/profiles/ProfilesTab.tsx @@ -30,7 +30,6 @@ export interface Profile { } interface CreateProfilePayload { - profile_id?: string; name: string; description?: string | null; criteria: JsonObject; @@ -39,7 +38,6 @@ interface CreateProfilePayload { } interface ProfileFormValues { - profile_id: string; name: string; description: string; criteria: JsonObject; @@ -78,12 +76,6 @@ function transformPayload(values: ProfileFormValues): CreateProfilePayload { is_active: values.is_active, }; - const profileId = values.profile_id.trim(); - - if (profileId) { - payload.profile_id = profileId; - } - const description = values.description.trim(); if (description) { @@ -109,7 +101,6 @@ const profileSchema = getSchema("profile"); function createDefaultValues(): ProfileFormValues { return { - profile_id: "", name: "", description: "", criteria: buildDefaultValue(profileSchema) as JsonObject, @@ -228,12 +219,7 @@ export function ProfilesTab() { const updateProfileMutation = useMutation({ mutationFn: ({ id, data }: { id: string; data: ProfileFormValues }) => { const payload = transformPayload(data); - const { profile_id: _ignored, ...body } = payload; - - return api.put, Profile>( - `/profiles/${id}`, - body - ); + return api.put(`/profiles/${id}`, payload); }, onSuccess: () => { queryClient.invalidateQueries({ queryKey: ["profiles"] }); @@ -285,18 +271,6 @@ export function ProfilesTab() {
-
- - -
-