diff --git a/backend/api_app/controllers/apps.py b/backend/api_app/controllers/apps.py index f036197..3c8a4c3 100644 --- a/backend/api_app/controllers/apps.py +++ b/backend/api_app/controllers/apps.py @@ -13,7 +13,7 @@ import pandas as pd from adscrawler import connection as write_conn from adscrawler.app_stores import apple, google, scrape_stores -from litestar import Controller, Response, get +from litestar import Controller, Response, get, post from litestar.background_tasks import BackgroundTask from litestar.exceptions import NotFoundException @@ -38,6 +38,7 @@ get_single_apps_adstxt, get_single_developer, get_total_counts, + insert_sdk_scan_request, search_apps, ) @@ -568,6 +569,13 @@ async def search_applestore(self: Self, search_term: str) -> AppGroup: ) return app_group + @post(path="/{store_id:str}/requestSDKScan") + async def request_sdk_scan(self: Self, store_id: str) -> Response: + """Request a new SDK scan for an app.""" + logger.info(f"Requesting SDK scan for {store_id}") + insert_sdk_scan_request(store_id) + return + COLLECTIONS = { "new_weekly": {"title": "New Apps this Week"}, diff --git a/backend/dbcon/connections.py b/backend/dbcon/connections.py index b2460a7..6123695 100644 --- a/backend/dbcon/connections.py +++ b/backend/dbcon/connections.py @@ -1,86 +1,86 @@ """Create SQLAlchemy database connection engine.""" -from typing import Self +from socket import gethostbyname -from sqlalchemy import create_engine +import sqlalchemy from config import CONFIG, get_logger logger = get_logger(__name__) +def get_host_ip(hostname: str) -> str: + """Convert hostname to IPv4 address if needed.""" + # Check if hostname is already an IPv4 address + if all(part.isdigit() and 0 <= int(part) <= 255 for part in hostname.split(".")): # noqa: PLR2004 + return hostname + ip_address = gethostbyname(hostname) + logger.info(f"Resolved {hostname} to {ip_address}") + return ip_address + + def open_ssh_tunnel(server_name: str): # noqa: ANN201 """Create SSH tunnel when working remotely.""" from sshtunnel import SSHTunnelForwarder + ssh_host = get_host_ip(CONFIG[server_name]["host"]) + + ssh_port = CONFIG[server_name].get("ssh_port", 22) + with SSHTunnelForwarder( - (CONFIG[server_name]["host"], 22), # Remote server IP and SSH port + (ssh_host, ssh_port), # Remote server IP and SSH port ssh_username=CONFIG[server_name]["os_user"], ssh_pkey=CONFIG[server_name].get("ssh_pkey", None), ssh_private_key_password=CONFIG[server_name].get("ssh_pkey_password", None), remote_bind_address=("127.0.0.1", 5432), ) as server: # PostgreSQL server IP and sever port on remote machine - logger.info(f"Start SSH tunnel to {server_name=}") logger.info(f"Opened SSH Tunnel {server_name=}") return server class PostgresCon: - """Class for managing the connection to postgres. + """Class for managing the connection to PostgreSQL.""" - Parameters - ---------- - my_db: String, passed on init, string name of db - my_env: String, passed on init, string name of env, 'staging' or 'prod' + def __init__(self, config_name: str, db_ip: str, db_port: str) -> None: + """Initialize the PostgreSQL connection. - """ + Args: + config_name (str): Corresponds to the server title in the config file. + db_ip (str): IP address of the database server. + db_port (str): Port number of the database server. - engine = None - db_name = None - db_pass = None - db_uri = None - db_user = None - - def __init__( - self: Self, - my_db: str, - db_ip: str | None = None, - db_port: str | None = None, - ) -> None: - """Initialize connection with ports and dbname.""" - self.db_name = my_db + """ + self.config_name = config_name + self.db_name = CONFIG[config_name]["db_name"] self.db_ip = db_ip self.db_port = db_port + self.engine: sqlalchemy.Engine + try: - self.db_user = CONFIG[self.db_name]["db_user"] - self.db_pass = CONFIG[self.db_name]["db_password"] - logger.info("Auth data loaded") - except Exception as error: - msg = f"Loading db_auth for {self.db_name}, error: {error}" - logger.exception(msg) - - def set_engine(self: Self) -> None: - """Set postgresql engine.""" + self.db_pass = CONFIG[self.config_name]["db_password"] + self.db_user = CONFIG[self.config_name]["db_user"] + except KeyError: + logger.exception(f"Loading db_auth for {self.config_name}") + raise + + def set_engine(self) -> None: + """Set up the SQLAlchemy engine.""" try: - self.db_uri = f"postgresql://{self.db_user}:{self.db_pass}" - self.db_uri += f"@{self.db_ip}:{self.db_port}/{self.db_name}" - self.engine = create_engine( - self.db_uri, - connect_args={ - "connect_timeout": 10, - "application_name": "appgoblin", - }, + db_login = f"postgresql://{self.db_user}:{self.db_pass}" + db_uri = f"{db_login}@{self.db_ip}:{self.db_port}/{self.db_name}" + logger.info(f"Adscrawler connecting to PostgreSQL {self.db_name}") + self.engine = sqlalchemy.create_engine( + db_uri, + connect_args={"connect_timeout": 10, "application_name": "adscrawler"}, ) - logger.info(f"Created PostgreSQL Engine {self.db_name}") - except Exception as error: - msg = ( - f"PostgresCon failed to connect to {self.db_name}@{self.db_ip} {error=}" + except Exception: + logger.exception( + f"Failed to connect {self.db_name} @ {self.db_ip}", ) - logger.exception(msg) - self.db_name = None + raise -def get_db_connection(server_name: str) -> PostgresCon: +def get_db_connection(server_config_name: str) -> PostgresCon: """Return PostgresCon class. to use class run server.set_engine() @@ -89,8 +89,8 @@ def get_db_connection(server_name: str) -> PostgresCon: Parameters server_name: str String of server name for parsing config file """ - server_ip, server_local_port = get_postgres_server_ips(server_name) - postgres_con = PostgresCon(server_name, server_ip, server_local_port) + server_ip, server_local_port = get_postgres_server_ips(server_config_name) + postgres_con = PostgresCon(server_config_name, server_ip, server_local_port) return postgres_con @@ -106,5 +106,5 @@ def get_postgres_server_ips(server_name: str) -> tuple[str, str]: ssh_server.start() db_port = str(ssh_server.local_bind_port) db_ip = "127.0.0.1" - logger.info(f"Connecting {db_ip=} {db_port=}") + logger.info(f"DB connection settings: {db_ip=} {db_port=}") return db_ip, db_port diff --git a/backend/dbcon/queries.py b/backend/dbcon/queries.py index 1d83230..ec12c62 100644 --- a/backend/dbcon/queries.py +++ b/backend/dbcon/queries.py @@ -76,6 +76,8 @@ def load_sql_file(file_name: str) -> str: QUERY_SITEMAP_APPS = load_sql_file("query_sitemap_apps.sql") QUERY_SITEMAP_COMPANIES = load_sql_file("query_sitemap_companies.sql") +INSERT_SDK_SCAN_REQUEST = load_sql_file("insert_sdk_scan_request.sql") + def get_recent_apps(collection: str, limit: int = 20) -> pd.DataFrame: """Get app collections by time.""" @@ -587,6 +589,17 @@ def get_sitemap_apps() -> pd.DataFrame: return df +def insert_sdk_scan_request(store_id: str) -> None: + """Insert a new sdk scan request.""" + logger.info(f"Inserting new sdk scan request: {store_id}") + + with DBCONWRITE.engine.connect() as connection: + connection.execute(INSERT_SDK_SCAN_REQUEST, {"store_id": store_id}) + connection.commit() + + logger.info("set db engine") DBCON = get_db_connection("madrone") DBCON.set_engine() +DBCONWRITE = get_db_connection("madrone-write") +DBCONWRITE.set_engine() diff --git a/frontend/package.json b/frontend/package.json index 3f2d378..48b28b5 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -19,6 +19,7 @@ "@sveltejs/adapter-node": "^5.2.6", "@sveltejs/kit": "^2.5.27", "@sveltejs/vite-plugin-svelte": "^5.0.0", + "@tailwindcss/forms": "^0.5.9", "@types/eslint": "^9.6.1", "@types/node": "^22.9.0", "@typescript-eslint/eslint-plugin": "^8.8.1", diff --git a/frontend/src/lib/RequestSDKScanButton.svelte b/frontend/src/lib/RequestSDKScanButton.svelte new file mode 100644 index 0000000..07a08ad --- /dev/null +++ b/frontend/src/lib/RequestSDKScanButton.svelte @@ -0,0 +1,39 @@ + + +
+ + + {#if myMessage} + +

{myMessage}

+ {/if} +

+ This will request a new SDK scan for this app. Scanning may take several days, or require manual + troubleshooting. Please reach out on Discord and we can help work on the scan. iOS is currently + not working well due to changes in Apple APIs and may not be possible. +

+
diff --git a/frontend/src/routes/apps/[id]/+page.server.ts b/frontend/src/routes/apps/[id]/+page.server.ts index ac8e01b..aab82f7 100644 --- a/frontend/src/routes/apps/[id]/+page.server.ts +++ b/frontend/src/routes/apps/[id]/+page.server.ts @@ -1,5 +1,24 @@ import type { PageServerLoad } from './$types.js'; +import type { Actions } from './$types'; + +export const actions = { + requestSDKScan: async (event) => { + const formData = await event.request.formData(); + const appId = formData.get('appId'); + console.log('requestSDKScan', appId); + + const response = await fetch(`http://localhost:8000/api/apps/${appId}/requestSDKScan`, { + method: 'POST' + }); + if (response.status === 200) { + return { success: true }; + } else { + return { success: false }; + } + } +} satisfies Actions; + function checkStatus(resp: Response, name: string) { if (resp.status === 200) { return resp.json(); diff --git a/frontend/src/routes/apps/[id]/+page.svelte b/frontend/src/routes/apps/[id]/+page.svelte index 834b730..b8682ee 100644 --- a/frontend/src/routes/apps/[id]/+page.svelte +++ b/frontend/src/routes/apps/[id]/+page.svelte @@ -1,6 +1,7 @@