📃 #12 — We should only bring down the statuses that don't exist in the SQLite database (#24)

Author: phildini

mastodon_to_sqlite/cli.py

@@ -165,7 +165,15 @@ def followings(db_path, auth):
     default="auth.json",
     help="Path to auth.json token file",
 )
-def statuses(db_path, auth):
+@click.option(
+    "-u",
+    "--update",
+    is_flag=True,
+    show_default=True,
+    default=False,
+    help="Update existing statuses",
+)
+def statuses(db_path, auth, update):
     """
     Save statuses for the authenticated user.
     """
@@ -177,8 +185,12 @@ def statuses(db_path, auth):
 
     service.save_accounts(db, [authenticated_account])
 
+    since_id = None
+    if update:
+        since_id = service.get_most_recent_status_id(db)
+
     with click.progressbar(
-        service.get_statuses(account_id, client),
+        service.get_statuses(account_id, client, since_id=since_id),
         label="Importing statuses",
         show_pos=True,
     ) as bar:

mastodon_to_sqlite/client.py

@@ -1,6 +1,6 @@
 import datetime
 from time import sleep
-from typing import Generator, Optional, Tuple
+from typing import Dict, Generator, Optional, Tuple
 
 from requests import PreparedRequest, Request, Response, Session
 from requests.auth import AuthBase
@@ -37,12 +37,15 @@ def request(
         self,
         method: str,
         path: str,
+        params: Optional[Dict[str, str]] = None,
         timeout: Optional[Tuple[int, int]] = None,
         **kwargs,
     ) -> Tuple[PreparedRequest, Response]:
         full_url = f"{self.api_url}/{path}"
 
-        request = Request(method=method.upper(), url=full_url, **kwargs)
+        request = Request(
+            method=method.upper(), url=full_url, params=params, **kwargs
+        )
         prepped = self.session.prepare_request(request)
         response = self.session.send(prepped, timeout=timeout)
 
@@ -52,14 +55,19 @@ def request_paginated(
         self,
         method: str,
         path: str,
+        params: Optional[Dict[str, str]] = None,
         timeout: Optional[Tuple[int, int]] = None,
         **kwargs,
     ) -> Generator[Tuple[PreparedRequest, Response], None, None]:
         next_path: Optional[str] = path
 
         while next_path is not None:
             request, response = self.request(
-                method=method, path=next_path, timeout=timeout, **kwargs
+                method=method,
+                path=next_path,
+                timeout=timeout,
+                params=params,
+                **kwargs,
             )
             yield request, response
 
@@ -82,6 +90,10 @@ def request_paginated(
             next_url = response.links["next"]["url"]
             next_path = next_url.replace(f"{self.api_url}/", "")
 
+            # Resetting the params because the next_path will provide the query
+            # parameters.
+            params = {}
+
     def accounts_verify_credentials(self) -> Tuple[PreparedRequest, Response]:
         return self.request("GET", "accounts/verify_credentials")
 
@@ -100,10 +112,17 @@ def accounts_following(
         )
 
     def accounts_statuses(
-        self, account_id: str
+        self,
+        account_id: str,
+        since_id: Optional[str] = None,
     ) -> Generator[Tuple[PreparedRequest, Response], None, None]:
+        params = {"limit": "40"}
+
+        if since_id is not None:
+            params["since_id"] = since_id
+
         return self.request_paginated(
-            "GET", f"accounts/{account_id}/statuses", params={"limit": "40"}
+            "GET", f"accounts/{account_id}/statuses", params=params
         )
 
     def bookmarks(

mastodon_to_sqlite/service.py

@@ -217,12 +217,14 @@ def save_accounts(
 
 
 def get_statuses(
-    account_id: str, client: MastodonClient
+    account_id: str, client: MastodonClient, since_id: Optional[str] = None
 ) -> Generator[List[Dict[str, Any]], None, None]:
     """
     Get authenticated account's statuses.
     """
-    for request, response in client.accounts_statuses(account_id):
+    for request, response in client.accounts_statuses(
+        account_id, since_id=since_id
+    ):
         yield response.json()
 
 
@@ -307,3 +309,20 @@ def save_activities(
         ),
         pk=("account_id", "activity", "status_id"),
     )
+
+
+def get_most_recent_status_id(db: Database) -> Optional[int]:
+    """
+    Get the most recent status ID from the SQLite database.
+    """
+    table = get_table("statuses", db=db)
+
+    row = next(
+        table.rows_where(order_by="created_at desc", select="id", limit=1),
+        None,
+    )
+
+    if row is None:
+        return None
+
+    return row["id"]

tests/fixtures.py

@@ -34,7 +34,7 @@
 
 STATUS_TWO = {
     "id": "2",
-    "created_at": "2021-12-20T19:46:29.073Z",
+    "created_at": "2021-12-20T20:46:29.073Z",
     "content": "Sleds are for suckers! Just ride on my gut!",
     "account": ACCOUNT_TWO,
     "bookmarked": True,

tests/test_services.py

@@ -111,3 +111,16 @@ def test_save_multiple_activity_types(mock_db):
     assert mock_db["status_activities"].count == 4
     for row in mock_db["status_activities"].rows:
         assert row["account_id"] == 42
+
+
+def test_get_most_recent_status_id(mock_db):
+    result = service.get_most_recent_status_id(mock_db)
+    assert result is None
+
+    status_one = fixtures.STATUS_ONE.copy()
+    status_two = fixtures.STATUS_TWO.copy()
+
+    service.save_statuses(mock_db, [status_one, status_two])
+
+    result = service.get_most_recent_status_id(mock_db)
+    assert result == int(status_two["id"])