Add form to request tracking

backend/api_app/controllers/apps.py

@@ -13,7 +13,7 @@
 import pandas as pd
 from adscrawler import connection as write_conn
 from adscrawler.app_stores import apple, google, scrape_stores
-from litestar import Controller, Response, get
+from litestar import Controller, Response, get, post
 from litestar.background_tasks import BackgroundTask
 from litestar.exceptions import NotFoundException
 
@@ -38,6 +38,7 @@
     get_single_apps_adstxt,
     get_single_developer,
     get_total_counts,
+    insert_sdk_scan_request,
     search_apps,
 )
 
@@ -568,6 +569,13 @@ async def search_applestore(self: Self, search_term: str) -> AppGroup:
             )
         return app_group
 
+    @post(path="/{store_id:str}/requestSDKScan")
+    async def request_sdk_scan(self: Self, store_id: str) -> Response:
+        """Request a new SDK scan for an app."""
+        logger.info(f"Requesting SDK scan for {store_id}")
+        insert_sdk_scan_request(store_id)
+        return
+
 
 COLLECTIONS = {
     "new_weekly": {"title": "New Apps this Week"},

backend/dbcon/connections.py

@@ -1,86 +1,86 @@
 """Create SQLAlchemy database connection engine."""
 
-from typing import Self
+from socket import gethostbyname
 
-from sqlalchemy import create_engine
+import sqlalchemy
 
 from config import CONFIG, get_logger
 
 logger = get_logger(__name__)
 
 
+def get_host_ip(hostname: str) -> str:
+    """Convert hostname to IPv4 address if needed."""
+    # Check if hostname is already an IPv4 address
+    if all(part.isdigit() and 0 <= int(part) <= 255 for part in hostname.split(".")):  # noqa: PLR2004
+        return hostname
+    ip_address = gethostbyname(hostname)
+    logger.info(f"Resolved {hostname} to {ip_address}")
+    return ip_address
+
+
 def open_ssh_tunnel(server_name: str):  # noqa: ANN201
     """Create SSH tunnel when working remotely."""
     from sshtunnel import SSHTunnelForwarder
 
+    ssh_host = get_host_ip(CONFIG[server_name]["host"])
+
+    ssh_port = CONFIG[server_name].get("ssh_port", 22)
+
     with SSHTunnelForwarder(
-        (CONFIG[server_name]["host"], 22),  # Remote server IP and SSH port
+        (ssh_host, ssh_port),  # Remote server IP and SSH port
         ssh_username=CONFIG[server_name]["os_user"],
         ssh_pkey=CONFIG[server_name].get("ssh_pkey", None),
         ssh_private_key_password=CONFIG[server_name].get("ssh_pkey_password", None),
         remote_bind_address=("127.0.0.1", 5432),
     ) as server:  # PostgreSQL server IP and sever port on remote machine
-        logger.info(f"Start SSH tunnel to {server_name=}")
         logger.info(f"Opened SSH Tunnel {server_name=}")
     return server
 
 
 class PostgresCon:
-    """Class for managing the connection to postgres.
+    """Class for managing the connection to PostgreSQL."""
 
-    Parameters
-    ----------
-        my_db: String, passed on init, string name of db
-        my_env: String, passed on init, string name of env, 'staging' or 'prod'
+    def __init__(self, config_name: str, db_ip: str, db_port: str) -> None:
+        """Initialize the PostgreSQL connection.
 
-    """
+        Args:
+            config_name (str): Corresponds to the server title in the config file.
+            db_ip (str): IP address of the database server.
+            db_port (str): Port number of the database server.
 
-    engine = None
-    db_name = None
-    db_pass = None
-    db_uri = None
-    db_user = None
-
-    def __init__(
-        self: Self,
-        my_db: str,
-        db_ip: str | None = None,
-        db_port: str | None = None,
-    ) -> None:
-        """Initialize connection with ports and dbname."""
-        self.db_name = my_db
+        """
+        self.config_name = config_name
+        self.db_name = CONFIG[config_name]["db_name"]
         self.db_ip = db_ip
         self.db_port = db_port
+        self.engine: sqlalchemy.Engine
+
         try:
-            self.db_user = CONFIG[self.db_name]["db_user"]
-            self.db_pass = CONFIG[self.db_name]["db_password"]
-            logger.info("Auth data loaded")
-        except Exception as error:
-            msg = f"Loading db_auth for {self.db_name}, error: {error}"
-            logger.exception(msg)
-
-    def set_engine(self: Self) -> None:
-        """Set postgresql engine."""
+            self.db_pass = CONFIG[self.config_name]["db_password"]
+            self.db_user = CONFIG[self.config_name]["db_user"]
+        except KeyError:
+            logger.exception(f"Loading db_auth for {self.config_name}")
+            raise
+
+    def set_engine(self) -> None:
+        """Set up the SQLAlchemy engine."""
         try:
-            self.db_uri = f"postgresql://{self.db_user}:{self.db_pass}"
-            self.db_uri += f"@{self.db_ip}:{self.db_port}/{self.db_name}"
-            self.engine = create_engine(
-                self.db_uri,
-                connect_args={
-                    "connect_timeout": 10,
-                    "application_name": "appgoblin",
-                },
+            db_login = f"postgresql://{self.db_user}:{self.db_pass}"
+            db_uri = f"{db_login}@{self.db_ip}:{self.db_port}/{self.db_name}"
+            logger.info(f"Adscrawler connecting to PostgreSQL {self.db_name}")
+            self.engine = sqlalchemy.create_engine(
+                db_uri,
+                connect_args={"connect_timeout": 10, "application_name": "adscrawler"},
             )
-            logger.info(f"Created PostgreSQL Engine {self.db_name}")
-        except Exception as error:
-            msg = (
-                f"PostgresCon failed to connect to {self.db_name}@{self.db_ip} {error=}"
+        except Exception:
+            logger.exception(
+                f"Failed to connect {self.db_name} @ {self.db_ip}",
             )
-            logger.exception(msg)
-            self.db_name = None
+            raise
 
 
-def get_db_connection(server_name: str) -> PostgresCon:
+def get_db_connection(server_config_name: str) -> PostgresCon:
     """Return PostgresCon class.
 
     to use class run server.set_engine()
@@ -89,8 +89,8 @@ def get_db_connection(server_name: str) -> PostgresCon:
     Parameters
        server_name: str String of server name for parsing config file
     """
-    server_ip, server_local_port = get_postgres_server_ips(server_name)
-    postgres_con = PostgresCon(server_name, server_ip, server_local_port)
+    server_ip, server_local_port = get_postgres_server_ips(server_config_name)
+    postgres_con = PostgresCon(server_config_name, server_ip, server_local_port)
     return postgres_con
 
 
@@ -106,5 +106,5 @@ def get_postgres_server_ips(server_name: str) -> tuple[str, str]:
         ssh_server.start()
         db_port = str(ssh_server.local_bind_port)
         db_ip = "127.0.0.1"
-    logger.info(f"Connecting {db_ip=} {db_port=}")
+    logger.info(f"DB connection settings: {db_ip=} {db_port=}")
     return db_ip, db_port

backend/dbcon/queries.py

@@ -76,6 +76,8 @@ def load_sql_file(file_name: str) -> str:
 QUERY_SITEMAP_APPS = load_sql_file("query_sitemap_apps.sql")
 QUERY_SITEMAP_COMPANIES = load_sql_file("query_sitemap_companies.sql")
 
+INSERT_SDK_SCAN_REQUEST = load_sql_file("insert_sdk_scan_request.sql")
+
 
 def get_recent_apps(collection: str, limit: int = 20) -> pd.DataFrame:
     """Get app collections by time."""
@@ -587,6 +589,17 @@ def get_sitemap_apps() -> pd.DataFrame:
     return df
 
 
+def insert_sdk_scan_request(store_id: str) -> None:
+    """Insert a new sdk scan request."""
+    logger.info(f"Inserting new sdk scan request: {store_id}")
+
+    with DBCONWRITE.engine.connect() as connection:
+        connection.execute(INSERT_SDK_SCAN_REQUEST, {"store_id": store_id})
+        connection.commit()
+
+
 logger.info("set db engine")
 DBCON = get_db_connection("madrone")
 DBCON.set_engine()
+DBCONWRITE = get_db_connection("madrone-write")
+DBCONWRITE.set_engine()

frontend/package.json

@@ -19,6 +19,7 @@
 		"@sveltejs/adapter-node": "^5.2.6",
 		"@sveltejs/kit": "^2.5.27",
 		"@sveltejs/vite-plugin-svelte": "^5.0.0",
+		"@tailwindcss/forms": "^0.5.9",
 		"@types/eslint": "^9.6.1",
 		"@types/node": "^22.9.0",
 		"@typescript-eslint/eslint-plugin": "^8.8.1",

frontend/src/lib/RequestSDKScanButton.svelte

@@ -0,0 +1,39 @@
+<script lang="ts">
+	import { enhance } from '$app/forms';
+	import { page } from '$app/state';
+
+	let myMessage = $state('');
+
+	function handleSubmit() {
+		return async ({ result }) => {
+			// Handle the form submission result
+			if (result.type === 'success') {
+				// You can add additional success handling here if needed
+				console.log('success');
+				myMessage = 'Successfully requested SDK scan!';
+			} else {
+				myMessage = 'Failed to request SDK scan.';
+			}
+		};
+	}
+</script>
+
+<form
+	class="mx-auto w-full max-w-md space-y-4"
+	method="POST"
+	action="?/requestSDKScan"
+	use:enhance={handleSubmit}
+>
+	<input type="hidden" name="appId" value={page.params.id} />
+	<button class="btn preset-tonal">Request New SDK Scan</button>
+	{#if myMessage}
+		<!-- this message is ephemeral; it exists because the page was rendered in
+	       response to a form submission. it will vanish if the user reloads -->
+		<p class="text-green-500">{myMessage}</p>
+	{/if}
+	<p>
+		This will request a new SDK scan for this app. Scanning may take several days, or require manual
+		troubleshooting. Please reach out on Discord and we can help work on the scan. iOS is currently
+		not working well due to changes in Apple APIs and may not be possible.
+	</p>
+</form>

frontend/src/routes/apps/[id]/+page.server.ts

@@ -1,5 +1,24 @@
 import type { PageServerLoad } from './$types.js';
 
+import type { Actions } from './$types';
+
+export const actions = {
+	requestSDKScan: async (event) => {
+		const formData = await event.request.formData();
+		const appId = formData.get('appId');
+		console.log('requestSDKScan', appId);
+
+		const response = await fetch(`http://localhost:8000/api/apps/${appId}/requestSDKScan`, {
+			method: 'POST'
+		});
+		if (response.status === 200) {
+			return { success: true };
+		} else {
+			return { success: false };
+		}
+	}
+} satisfies Actions;
+
 function checkStatus(resp: Response, name: string) {
 	if (resp.status === 200) {
 		return resp.json();

frontend/src/routes/apps/[id]/+page.svelte

@@ -1,6 +1,7 @@
 <script lang="ts">
 	import ExternalLinkSvg from '$lib/svg/ExternalLinkSVG.svelte';
 	import ManifestItemList from '$lib/ManifestItemList.svelte';
+	import RequestSDKScanButton from '$lib/RequestSDKScanButton.svelte';
 	import type { AppFullDetails } from '../../../types';
 	import AppDetails from '$lib/RatingInstallsLarge.svelte';
 	import AppPlot from '$lib/AppPlot.svelte';
@@ -244,24 +245,29 @@
 							SDK Tracking Status
 						{/snippet}
 						<div class="space-y-2 p-2">
-							<div class="flex items-center gap-2">
-								<span class="font-medium">Successful Last Crawled:</span>
-								<span>{myapp.sdk_successful_last_crawled}</span>
-							</div>
-							<div class="flex items-center gap-2">
-								<span class="font-medium">Last Crawled:</span>
-								<span>{myapp.sdk_last_crawled}</span>
-							</div>
-							<div class="flex items-center gap-2">
-								<span class="font-medium">Last Crawl Status:</span>
-								<span
-									class={myapp.sdk_crawl_result == 1
-										? 'text-success-900-100'
-										: 'text-error-900-100'}
-								>
-									{myapp.sdk_crawl_result == 1 ? 'Success' : 'Failed'}
-								</span>
-							</div>
+							{#if myapp.sdk_last_crawled}
+								<div class="flex items-center gap-2">
+									<span class="font-medium">Successful Last Crawled:</span>
+									<span>{myapp.sdk_successful_last_crawled}</span>
+								</div>
+								<div class="flex items-center gap-2">
+									<span class="font-medium">Last Crawled:</span>
+									<span>{myapp.sdk_last_crawled}</span>
+								</div>
+								<div class="flex items-center gap-2">
+									<span class="font-medium">Last Crawl Status:</span>
+									<span
+										class={myapp.sdk_crawl_result == 1
+											? 'text-success-900-100'
+											: 'text-error-900-100'}
+									>
+										{myapp.sdk_crawl_result == 1 ? 'Success' : 'Failed'}
+									</span>
+								</div>
+							{:else}
+								App not yet analyzed for SDKs.
+								<RequestSDKScanButton />
+							{/if}
 						</div>
 					</WhiteCard>
 					<div class="ml-auto">
@@ -450,7 +456,7 @@
 					{/if}
 				{/if}
 			{/await}
-			<!-- <button class="btn preset-tonal">Request New SDK Scan</button> -->
+			<RequestSDKScanButton />
 		</div>
 	</div>
 </section>

frontend/tailwind.config.ts

@@ -1,5 +1,6 @@
 import { join } from 'path';
 import type { Config } from 'tailwindcss';
+import forms from '@tailwindcss/forms';
 
 import { skeleton } from '@skeletonlabs/skeleton/plugin';
 import * as themes from '@skeletonlabs/skeleton/themes';
@@ -14,6 +15,7 @@ const config = {
 		extend: {}
 	},
 	plugins: [
+		forms,
 		skeleton({
 			themes: [themes.nouveau, themes.catppuccin]
 		})