fix date generator, update tests, update version number, relative imports for gui

Date generator assumed that there was a fixed window of missing data, but actually, this is a changing window. Downloader now gives a warning about missing bid tables when they can't be downloaded.

Author: nick-gorman

.gitignore

@@ -27,3 +27,4 @@ NEMOSIS.exe
 nemosis/smoke_tests.py
 nemosis/check_new_bid_table_functionality.py
 *.pyc
+smoke.py

e.py

@@ -120,21 +120,6 @@ def bid_table_gen(start_time, end_time):
                         and year == end_year
                     ):
                         continue
-                    if int(year) == 2021 and int(month) == 4 and int(day) == 1:
-                        logger.warning(
-                            "Offer data for 2021/04/01 is known to be missing from the AEMO public \n"
-                            "archive, explicitly skipping. This file would also contain data for the first 4 hr of \n"
-                            + "2021/04/02 so that data will also be missing from the returned dataframe."
-                        )
-                    else:
-                        yield str(year), month, str(day).zfill(2), None
-
+                    yield str(year), month, str(day).zfill(2), None
             else:
-                if int(year) == 2021 and int(month) == 3:
-                    logger.warning(
-                        "Offer data for March 2021 is known to be missing from the  AEMO public \n"
-                        "archive, explicitly skipping. This file would also contain data for the first 4 hr of \n"
-                        + "2021/04/01 so that data will also be missing from the returned dataframe."
-                    )
-                else:
-                    yield str(year), month, None, None
+                yield str(year), month, None, None

nemosis/date_generators.py

@@ -38,10 +38,15 @@ def run_bid_tables(year, month, day, index, filename_stub, down_load_to):
     if day is None:
         run(year, month, day, index, filename_stub, down_load_to)
     else:
-        _download_and_unpack_bid_move_complete_files(
-            year, month, day, index, filename_stub, down_load_to
-        )
-
+        try:
+            _download_and_unpack_bid_move_complete_files(
+                year, month, day, index, filename_stub, down_load_to
+            )
+        except Exception:
+            logger.warning(f"{filename_stub} not downloaded. This is likely because this file is not being hosted \n" +
+                           "online by AEMO. You can check this url to confirm: \n" +
+                           "https://www.nemweb.com.au/REPORTS/Archive/Bidmove_Complete/. If the file is available but \n"
+                           "this warning persists please contact the NEMOSIS maintainers.")
 
 def _download_and_unpack_bid_move_complete_files(
     year, month, day, index, filename_stub, down_load_to

nemosis/downloader.py

@@ -1,6 +1,6 @@
-from . import rows
-from . import defaults
-from . import data_fetch_methods
+from nemosis import rows
+from nemosis import defaults
+from nemosis import data_fetch_methods
 import pandas as pd
 import tkinter as tk
 import tkinter.ttk as ttk

nemosis/gui.py

@@ -5,7 +5,7 @@
 
 setuptools.setup(
     name="nemosis",
-    version="3.1.0",
+    version="3.2.0",
     author="Nicholas Gorman, Abhijith Prakash",
     author_email="n.gorman305@gmail.com",
     description="A tool for accessing AEMO data.",

setup.py

@@ -185,7 +185,6 @@ def test_change_from_months_to_days(self):
         end_time = datetime.strptime("2021/04/03 00:00:00", "%Y/%m/%d %H:%M:%S")
         gen = date_generators.bid_table_gen(start_time, end_time)
         times = [(year, month, day, index) for year, month, day, index in gen]
-        # Note we expect the 1st of april to be skipped
         self.assertEqual(times[0][0], "2021")
         self.assertEqual(times[0][1], "01")
         self.assertEqual(times[0][2], None)
@@ -194,49 +193,23 @@ def test_change_from_months_to_days(self):
         self.assertEqual(times[1][1], "02")
         self.assertEqual(times[1][2], None)
         self.assertEqual(times[1][3], None)
-        # Data for march and the first of april is missing from the AEMO website so we don't generate the dates
-        # for these times.
         self.assertEqual(times[2][0], "2021")
-        self.assertEqual(times[2][1], "04")
-        self.assertEqual(times[2][2], "02")
+        self.assertEqual(times[2][1], "03")
+        self.assertEqual(times[2][2], None)
         self.assertEqual(times[2][3], None)
         self.assertEqual(times[3][0], "2021")
         self.assertEqual(times[3][1], "04")
-        self.assertEqual(times[3][2], "03")
+        self.assertEqual(times[3][2], "01")
         self.assertEqual(times[3][3], None)
-        self.assertEqual(len(times), 4)
-
-    def test_day_given_in_april_2021(self):
-        start_time = datetime.strptime("2021/04/01 00:00:00", "%Y/%m/%d %H:%M:%S")
-        end_time = datetime.strptime("2021/04/03 00:00:00", "%Y/%m/%d %H:%M:%S")
-        gen = date_generators.bid_table_gen(start_time, end_time)
-        times = [(year, month, day, index) for year, month, day, index in gen]
-        # Note we expect the 1st of april to be skipped
-        self.assertEqual(times[0][0], "2021")
-        self.assertEqual(times[0][1], "04")
-        self.assertEqual(times[0][2], "02")
-        self.assertEqual(times[0][3], None)
-        self.assertEqual(times[1][0], "2021")
-        self.assertEqual(times[1][1], "04")
-        self.assertEqual(times[1][2], "03")
-        self.assertEqual(times[1][3], None)
-        self.assertEqual(len(times), 2)
-
-    def test_include_previous_market_day(self):
-        start_time = datetime.strptime("2021/05/10 01:00:00", "%Y/%m/%d %H:%M:%S")
-        end_time = datetime.strptime("2021/05/10 05:00:00", "%Y/%m/%d %H:%M:%S")
-        gen = date_generators.bid_table_gen(start_time, end_time)
-        times = [(year, month, day, index) for year, month, day, index in gen]
-        # Note we expect the 1st of april to be skipped
-        self.assertEqual(times[0][0], "2021")
-        self.assertEqual(times[0][1], "05")
-        self.assertEqual(times[0][2], "09")
-        self.assertEqual(times[0][3], None)
-        self.assertEqual(times[1][0], "2021")
-        self.assertEqual(times[1][1], "05")
-        self.assertEqual(times[1][2], "10")
-        self.assertEqual(times[1][3], None)
-        self.assertEqual(len(times), 2)
+        self.assertEqual(times[4][0], "2021")
+        self.assertEqual(times[4][1], "04")
+        self.assertEqual(times[4][2], "02")
+        self.assertEqual(times[4][3], None)
+        self.assertEqual(times[5][0], "2021")
+        self.assertEqual(times[5][1], "04")
+        self.assertEqual(times[5][2], "03")
+        self.assertEqual(times[5][3], None)
+        self.assertEqual(len(times), 6)
 
     def test_include_previous_month_if_1st_market_day_of_month(self):
         start_time = datetime.strptime("2021/05/01 05:00:00", "%Y/%m/%d %H:%M:%S")