Skip to content

Commit 3ffd3e8

Browse files
pre-commit-ci[bot]pre-commit-ci[bot]
committed
[pre-commit.ci] auto fixes from pre-commit.com hooks
for more information, see https://pre-commit.ci
1 parent 87072a7 commit 3ffd3e8

36 files changed

+3214
-1494
lines changed

‎.gitignore‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -162,4 +162,4 @@ ciccada/
162162

163163

164164
*.csv
165-
*.png
165+
*.png

‎BOM_NCI/Get_ALL_postcodes_ABS.ipynb‎

Lines changed: 23 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -8,12 +8,14 @@
88
"outputs": [],
99
"source": [
1010
"import sys\n",
11-
"sys.path.append('../') \n",
12-
"from visualisation import *\n",
13-
"import xarray as xr\n",
11+
"\n",
12+
"sys.path.append(\"../\")\n",
1413
"import dask\n",
1514
"import geopandas as gpd\n",
15+
"import xarray as xr\n",
1616
"from shapely.geometry import Point\n",
17+
"\n",
18+
"from visualisation import *\n",
1719
"# crs = EPSG:4326 (WGS 84)"
1820
]
1921
},
@@ -36,7 +38,7 @@
3638
],
3739
"source": [
3840
"bom_path = \"/home/hossein/CICCADA/BOM_NCI/2023/01/01/\"\n",
39-
"files = glob(bom_path+\"*.nc\")\n",
41+
"files = glob(bom_path + \"*.nc\")\n",
4042
"len(files)"
4143
]
4244
},
@@ -57,13 +59,13 @@
5759
"source": [
5860
"df = [xr.open_dataset(file).to_dataframe() for file in files[:15]]\n",
5961
"df = pd.concat(df, axis=0).reset_index(drop=False)\n",
60-
"df = df.dropna(subset='direct_normal_irradiance').reset_index(drop=True)\n",
61-
"df['julian_date'] = pd.to_datetime(df['julian_date'], origin='julian', unit='D')\n",
62-
"df = df[['latitude', 'longitude']].drop_duplicates().reset_index(drop=True)\n",
62+
"df = df.dropna(subset=\"direct_normal_irradiance\").reset_index(drop=True)\n",
63+
"df[\"julian_date\"] = pd.to_datetime(df[\"julian_date\"], origin=\"julian\", unit=\"D\")\n",
64+
"df = df[[\"latitude\", \"longitude\"]].drop_duplicates().reset_index(drop=True)\n",
6365
"# df = df.query(f\"latitude >= -35 & latitude <= -34.6 & longitude >= 138.5 & longitude <= 138.8\").reset_index(drop=True)\n",
64-
"df['geometry'] = [Point(x,y) for x,y in zip(df['longitude'], df['latitude'])]\n",
65-
"geo_list = df['geometry'].unique()\n",
66-
"print('len(geo_list): ', len(geo_list))"
66+
"df[\"geometry\"] = [Point(x, y) for x, y in zip(df[\"longitude\"], df[\"latitude\"])]\n",
67+
"geo_list = df[\"geometry\"].unique()\n",
68+
"print(\"len(geo_list): \", len(geo_list))"
6769
]
6870
},
6971
{
@@ -73,7 +75,9 @@
7375
"metadata": {},
7476
"outputs": [],
7577
"source": [
76-
"gdf = gpd.GeoDataFrame(df[['longitude', 'latitude', 'geometry']], geometry='geometry', crs='EPSG:4326') # assuming WGS84"
78+
"gdf = gpd.GeoDataFrame(\n",
79+
" df[[\"longitude\", \"latitude\", \"geometry\"]], geometry=\"geometry\", crs=\"EPSG:4326\"\n",
80+
") # assuming WGS84"
7781
]
7882
},
7983
{
@@ -83,8 +87,8 @@
8387
"metadata": {},
8488
"outputs": [],
8589
"source": [
86-
"gdf_postcodes = gpd.read_file('POA_2021_AUST_GDA2020_SHP/POA_2021_AUST_GDA2020.shp')\n",
87-
"gdf_postcodes = gdf_postcodes.to_crs('EPSG:4326') # Ensure same CRS"
90+
"gdf_postcodes = gpd.read_file(\"POA_2021_AUST_GDA2020_SHP/POA_2021_AUST_GDA2020.shp\")\n",
91+
"gdf_postcodes = gdf_postcodes.to_crs(\"EPSG:4326\") # Ensure same CRS"
8892
]
8993
},
9094
{
@@ -153,10 +157,7 @@
153157
"outputs": [],
154158
"source": [
155159
"gdf_joined = gpd.sjoin(\n",
156-
" gdf,\n",
157-
" gdf_postcodes[['POA_CODE21', 'geometry']],\n",
158-
" how='left',\n",
159-
" predicate='within'\n",
160+
" gdf, gdf_postcodes[[\"POA_CODE21\", \"geometry\"]], how=\"left\", predicate=\"within\"\n",
160161
")"
161162
]
162163
},
@@ -167,7 +168,7 @@
167168
"metadata": {},
168169
"outputs": [],
169170
"source": [
170-
"gdf_joined.drop(columns=['index_right'], inplace=True)\n",
171+
"gdf_joined.drop(columns=[\"index_right\"], inplace=True)\n",
171172
"gdf_joined = gdf_joined.dropna().reset_index(drop=True)"
172173
]
173174
},
@@ -178,7 +179,7 @@
178179
"metadata": {},
179180
"outputs": [],
180181
"source": [
181-
"gdf_joined.to_csv('bom_postcodes_points.csv', index=False)"
182+
"gdf_joined.to_csv(\"bom_postcodes_points.csv\", index=False)"
182183
]
183184
},
184185
{
@@ -188,7 +189,7 @@
188189
"metadata": {},
189190
"outputs": [],
190191
"source": [
191-
"gdf_postcodes['geometry'][0]"
192+
"gdf_postcodes[\"geometry\"][0]"
192193
]
193194
},
194195
{
@@ -265,7 +266,7 @@
265266
],
266267
"source": [
267268
"fig, ax = plt.subplots()\n",
268-
"gdf_postcodes.plot(ax=ax, facecolor='none', edgecolor='black')\n"
269+
"gdf_postcodes.plot(ax=ax, facecolor=\"none\", edgecolor=\"black\")"
269270
]
270271
},
271272
{
@@ -275,7 +276,7 @@
275276
"metadata": {},
276277
"outputs": [],
277278
"source": [
278-
"gdf.plot(ax=ax, color='red', markersize=2)\n",
279+
"gdf.plot(ax=ax, color=\"red\", markersize=2)\n",
279280
"plt.show()"
280281
]
281282
}

‎BOM_NCI/Get_ALL_postcodes_GNAF.ipynb‎

Lines changed: 38 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -8,13 +8,15 @@
88
"outputs": [],
99
"source": [
1010
"import sys\n",
11-
"sys.path.append('../') \n",
12-
"from visualisation import *\n",
13-
"import xarray as xr\n",
11+
"\n",
12+
"sys.path.append(\"../\")\n",
1413
"import dask\n",
1514
"import geopandas as gpd\n",
15+
"import xarray as xr\n",
1616
"from shapely.geometry import Point\n",
1717
"from sklearn.neighbors import KDTree\n",
18+
"\n",
19+
"from visualisation import *\n",
1820
"# crs = EPSG:4326 (WGS 84)"
1921
]
2022
},
@@ -37,7 +39,7 @@
3739
],
3840
"source": [
3941
"bom_path = \"/home/hossein/CICCADA/BOM_NCI/2023/01/01/\"\n",
40-
"files = glob(bom_path+\"*.nc\")\n",
42+
"files = glob(bom_path + \"*.nc\")\n",
4143
"len(files)"
4244
]
4345
},
@@ -50,9 +52,9 @@
5052
"source": [
5153
"df = [xr.open_dataset(file).to_dataframe() for file in files[:15]]\n",
5254
"df = pd.concat(df, axis=0).reset_index(drop=False)\n",
53-
"df = df.dropna(subset='direct_normal_irradiance').reset_index(drop=True)\n",
54-
"df['julian_date'] = pd.to_datetime(df['julian_date'], origin='julian', unit='D')\n",
55-
"df = df[['latitude', 'longitude']].drop_duplicates().reset_index(drop=True)"
55+
"df = df.dropna(subset=\"direct_normal_irradiance\").reset_index(drop=True)\n",
56+
"df[\"julian_date\"] = pd.to_datetime(df[\"julian_date\"], origin=\"julian\", unit=\"D\")\n",
57+
"df = df[[\"latitude\", \"longitude\"]].drop_duplicates().reset_index(drop=True)"
5658
]
5759
},
5860
{
@@ -113,9 +115,13 @@
113115
"metadata": {},
114116
"outputs": [],
115117
"source": [
116-
"SA_STREET_LOCALITY_POINT_psv = pd.read_csv(glob(f\"{naf_path}SA_STREET_LOCALITY_POINT_psv.psv\")[0], sep='|', low_memory=False).dropna(axis=1)\n",
117-
"SA_ADDRESS_DETAIL_psv = pd.read_csv(glob(f\"{naf_path}SA_ADDRESS_DETAIL_psv.psv\")[0], sep='|', low_memory=False).dropna(axis=1)\n",
118-
"# SA_ADDRESS_DETAIL_psv\n"
118+
"SA_STREET_LOCALITY_POINT_psv = pd.read_csv(\n",
119+
" glob(f\"{naf_path}SA_STREET_LOCALITY_POINT_psv.psv\")[0], sep=\"|\", low_memory=False\n",
120+
").dropna(axis=1)\n",
121+
"SA_ADDRESS_DETAIL_psv = pd.read_csv(\n",
122+
" glob(f\"{naf_path}SA_ADDRESS_DETAIL_psv.psv\")[0], sep=\"|\", low_memory=False\n",
123+
").dropna(axis=1)\n",
124+
"# SA_ADDRESS_DETAIL_psv"
119125
]
120126
},
121127
{
@@ -292,7 +298,9 @@
292298
"metadata": {},
293299
"outputs": [],
294300
"source": [
295-
"a = pd.read_csv(glob(f\"{naf_path}SA_ADDRESS_DETAIL_psv.psv\")[0], sep='|', low_memory=False)"
301+
"a = pd.read_csv(\n",
302+
" glob(f\"{naf_path}SA_ADDRESS_DETAIL_psv.psv\")[0], sep=\"|\", low_memory=False\n",
303+
")"
296304
]
297305
},
298306
{
@@ -334,7 +342,7 @@
334342
"metadata": {},
335343
"outputs": [],
336344
"source": [
337-
"5035 in SA_ADDRESS_DETAIL_psv['POSTCODE'].unique()"
345+
"5035 in SA_ADDRESS_DETAIL_psv[\"POSTCODE\"].unique()"
338346
]
339347
},
340348
{
@@ -344,7 +352,7 @@
344352
"metadata": {},
345353
"outputs": [],
346354
"source": [
347-
"SA_ADDRESS_DETAIL_psv['POSTCODE'].unique().shape"
355+
"SA_ADDRESS_DETAIL_psv[\"POSTCODE\"].unique().shape"
348356
]
349357
},
350358
{
@@ -365,8 +373,14 @@
365373
}
366374
],
367375
"source": [
368-
"locaility_points = SA_STREET_LOCALITY_POINT_psv[['STREET_LOCALITY_PID', 'LONGITUDE', 'LATITUDE']].merge(SA_ADDRESS_DETAIL_psv[['STREET_LOCALITY_PID', 'POSTCODE']].drop_duplicates(), on='STREET_LOCALITY_PID', how='left')\n",
369-
"locaility_points.drop(columns=['STREET_LOCALITY_PID'], inplace=True)\n",
376+
"locaility_points = SA_STREET_LOCALITY_POINT_psv[\n",
377+
" [\"STREET_LOCALITY_PID\", \"LONGITUDE\", \"LATITUDE\"]\n",
378+
"].merge(\n",
379+
" SA_ADDRESS_DETAIL_psv[[\"STREET_LOCALITY_PID\", \"POSTCODE\"]].drop_duplicates(),\n",
380+
" on=\"STREET_LOCALITY_PID\",\n",
381+
" how=\"left\",\n",
382+
")\n",
383+
"locaility_points.drop(columns=[\"STREET_LOCALITY_PID\"], inplace=True)\n",
370384
"locaility_points.dropna(inplace=True)\n",
371385
"locaility_points.columns"
372386
]
@@ -388,8 +402,8 @@
388402
"metadata": {},
389403
"outputs": [],
390404
"source": [
391-
"postcode_coords = locaility_points[['LATITUDE', 'LONGITUDE']].to_numpy()\n",
392-
"kdtree = KDTree(postcode_coords, metric='euclidean')"
405+
"postcode_coords = locaility_points[[\"LATITUDE\", \"LONGITUDE\"]].to_numpy()\n",
406+
"kdtree = KDTree(postcode_coords, metric=\"euclidean\")"
393407
]
394408
},
395409
{
@@ -414,9 +428,11 @@
414428
"metadata": {},
415429
"outputs": [],
416430
"source": [
417-
"df['nearest_postcode'] = locaility_points.iloc[nearest_indices]['POSTCODE'].values\n",
431+
"df[\"nearest_postcode\"] = locaility_points.iloc[nearest_indices][\"POSTCODE\"].values\n",
418432
"\n",
419-
"df['distance_km'] = nearest_distances*111 # Rough conversion factor for degrees to kilometers"
433+
"df[\"distance_km\"] = (\n",
434+
" nearest_distances * 111\n",
435+
") # Rough conversion factor for degrees to kilometers"
420436
]
421437
},
422438
{
@@ -447,7 +463,7 @@
447463
}
448464
],
449465
"source": [
450-
"df0['nearest_postcode'].unique().shape"
466+
"df0[\"nearest_postcode\"].unique().shape"
451467
]
452468
},
453469
{
@@ -468,7 +484,7 @@
468484
}
469485
],
470486
"source": [
471-
"5035 in df0['nearest_postcode'].unique()"
487+
"5035 in df0[\"nearest_postcode\"].unique()"
472488
]
473489
},
474490
{
@@ -478,7 +494,7 @@
478494
"metadata": {},
479495
"outputs": [],
480496
"source": [
481-
"df0.to_csv('bom_postcodes_points.csv', index=False)"
497+
"df0.to_csv(\"bom_postcodes_points.csv\", index=False)"
482498
]
483499
}
484500
],

‎BOM_NCI/describe_bom_data.ipynb‎

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -7,10 +7,13 @@
77
"outputs": [],
88
"source": [
99
"import sys\n",
10-
"sys.path.append('../') \n",
11-
"from visualisation import *\n",
10+
"\n",
11+
"sys.path.append(\"../\")\n",
12+
"import concurrent.futures\n",
13+
"\n",
1214
"import xarray as xr\n",
13-
"import concurrent.futures"
15+
"\n",
16+
"from visualisation import *"
1417
]
1518
},
1619
{
@@ -31,7 +34,7 @@
3134
],
3235
"source": [
3336
"bom_path = \"/home/hossein/CICCADA/BOM_NCI/2023/01/01/\"\n",
34-
"files = glob(bom_path+\"*.nc\")\n",
37+
"files = glob(bom_path + \"*.nc\")\n",
3538
"len(files)"
3639
]
3740
},
@@ -47,7 +50,7 @@
4750
"# print(ds)\n",
4851
"\n",
4952
"# List all variables\n",
50-
"# print(ds.variables)\n"
53+
"# print(ds.variables)"
5154
]
5255
},
5356
{
@@ -64,7 +67,7 @@
6467
}
6568
],
6669
"source": [
67-
"print(ds['latitude'].attrs)\n"
70+
"print(ds[\"latitude\"].attrs)"
6871
]
6972
},
7073
{
@@ -201,12 +204,12 @@
201204
}
202205
],
203206
"source": [
204-
"for key in ('quality_mask','cloud_type'):\n",
207+
"for key in (\"quality_mask\", \"cloud_type\"):\n",
205208
" print(ds[key].long_name)\n",
206209
" print(ds[key].flag_meanings)\n",
207210
" print(ds[key].flag_values)\n",
208211
" print(ds[key].comment)\n",
209-
" print('---------------------------------')"
212+
" print(\"---------------------------------\")"
210213
]
211214
}
212215
],

0 commit comments

Comments
 (0)