Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fixed normalization of oos skus #15

Merged
merged 18 commits into from
Sep 4, 2024
Merged

Fixed normalization of oos skus #15

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
b064f18
fixed error in selection of time_sku features
majoma7 Sep 1, 2024
22d5a2a
Merge branch 'main' into minor_fixes
majoma7 Sep 1, 2024
e7beb9d
fixed error in force save agent
majoma7 Sep 1, 2024
f7f7021
Merge branch 'main' into minor_fixes
majoma7 Sep 1, 2024
b58322c
fixed parameter name for dataset
majoma7 Sep 2, 2024
32d63e2
accelerated getitem method
majoma7 Sep 2, 2024
7a5babb
Merge branch 'main' into minor_fixes
majoma7 Sep 2, 2024
bdb9687
minor fix in predict function
majoma7 Sep 2, 2024
11fbb5d
minor fix in predict function
majoma7 Sep 2, 2024
b04aaa8
Merge branch 'main' into minor_fixes
majoma7 Sep 2, 2024
4ca6a13
enables out-of-sample experiments
majoma7 Sep 3, 2024
f297ea4
Merge branch 'main' into minor_fixes
majoma7 Sep 3, 2024
9b6d1e2
print steps while testing
majoma7 Sep 4, 2024
2d88574
merge
majoma7 Sep 4, 2024
6a90673
fixed error in normalization for oos products
majoma7 Sep 4, 2024
8030f0d
fixed error in normalizing oos demand
majoma7 Sep 4, 2024
1e57265
Merge branch 'main' into minor_fixes
majoma7 Sep 4, 2024
cf9de2f
fixed error in normalization of oos SKUs
majoma7 Sep 4, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 13 additions & 11 deletions ddopnew/dataloaders/tabular.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -861,35 +861,32 @@ def normalize_demand_and_features_out_of_sample(self,
self.demand_lag_out_of_sample_test = self.demand_out_of_sample_test.copy()
self.demand_lag_out_of_sample_val = self.demand_out_of_sample_val.copy()

print(np.sum(self.demand_lag_out_of_sample_val))
print(np.sum(self.demand_lag_out_of_sample_test))
print(np.sum(self.demand_out_of_sample_val))
print(np.sum(self.demand_out_of_sample_test))


# Normalize demand targets
if self.demand_normalization != 'no_normalization':
# Normalizing per SKU on time dimension
self.scaler_out_of_sample_test_demand.fit(self.demand_out_of_sample_test[:self.train_index_end+1])
transformed_demand = self.scaler_out_of_sample_test_demand.transform(self.demand_lag_out_of_sample_test)
self.demand_lag_out_of_sample_test.iloc[:,:] = transformed_demand
self.demand_out_of_sample_test.iloc[:,:] = transformed_demand

self.scaler_out_of_sample_val_demand.fit(self.demand_out_of_sample_val[:self.train_index_end+1])
transformed_demand = self.scaler_out_of_sample_val_demand.transform(self.demand_lag_out_of_sample_val)
self.demand_lag_out_of_sample_val.iloc[:,:] = transformed_demand
self.demand_out_of_sample_val.iloc[:,:] = transformed_demand

# Set unit size for demand targets
if self.demand_unit_size != None:
self.demand_lag_out_of_sample_test = np.round(self.demand_lag_out_of_sample_test, self.demand_unit_size)
self.demand_lag_out_of_sample_val = np.round(self.demand_lag_out_of_sample_val, self.demand_unit_size)
self.demand_out_of_sample_test = np.round(self.demand_out_of_sample_test, self.demand_unit_size)
self.demand_out_of_sample_val = np.round(self.demand_out_of_sample_val, self.demand_unit_size)

# If separate normalization for lag demand, normalize it
if self.lag_demand_normalization != self.demand_normalization:
if self.lag_demand_normalization != 'no_normalization':

self.demand_lag_out_of_sample_test = self.demand_out_of_sample_test.copy()
self.scaler_out_of_sample_test_demand_lag.fit(self.demand_lag_out_of_sample_test[:self.train_index_end+1])
transformed_demand_lag = self.scaler_out_of_sample_test_demand_lag.transform(self.demand_lag_out_of_sample_test)
self.demand_lag_out_of_sample_test.iloc[:,:] = transformed_demand_lag

self.demand_lag_out_of_sample_val = self.demand_out_of_sample_val.copy()
self.scaler_out_of_sample_val_demand_lag.fit(self.demand_lag_out_of_sample_val[:self.train_index_end+1])
transformed_demand_lag = self.scaler_out_of_sample_val_demand_lag.transform(self.demand_lag_out_of_sample_val)
self.demand_lag_out_of_sample_val.iloc[:,:] = transformed_demand_lag
Expand All @@ -899,7 +896,6 @@ def normalize_demand_and_features_out_of_sample(self,
self.demand_lag_out_of_sample_test = self.demand_out_of_sample_test.copy()
self.demand_lag_out_of_sample_val = self.demand_out_of_sample_val.copy()


if self.SKU_features is not None:
logging.info("--Normalizing SKU features")
# Normalizing across SKUs, no time dimension present
Expand Down Expand Up @@ -936,6 +932,12 @@ def normalize_demand_and_features_out_of_sample(self,

self.normalized_out_of_sample_SKUs = True

print(self.demand_out_of_sample_test)
print(self.demand_out_of_sample_val)

print(self.demand_lag_out_of_sample_test)
print(self.demand_lag_out_of_sample_val)

else:
raise NotImplementedError('Training data can only normalized during initialization - later normlization not implemented yet')

Expand Down
114 changes: 58 additions & 56 deletions nbs/10_dataloaders/12_tabular_dataloaders.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -588,7 +588,7 @@
"name": "stdout",
"output_type": "stream",
"text": [
"sample: [ 0.21393808 -0.43202782] [-0.19715301]\n",
"sample: [ 0.48934242 -1.5249851 ] [-4.20898833]\n",
"sample shape Y: (1,)\n",
"length: 100\n"
]
Expand Down Expand Up @@ -627,28 +627,28 @@
"length train: 6 length val: 2 length test: 2\n",
"\n",
"### Data from train set ###\n",
"idx: 0 data: [-0.68057372 -0.94303927] [-4.48274392]\n",
"idx: 1 data: [-0.4473647 1.27208487] [1.70464728]\n",
"idx: 2 data: [-0.4682478 2.49858032] [6.32016286]\n",
"idx: 3 data: [-0.89608283 0.06252056] [0.42949332]\n",
"idx: 4 data: [-0.26576938 -0.77356473] [-2.50971628]\n",
"idx: 5 data: [1.51339474 0.83226233] [4.76266322]\n",
"idx: 0 data: [-0.03919358 -1.81524105] [-5.45285373]\n",
"idx: 1 data: [-0.33238759 -0.25239502] [-2.28257421]\n",
"idx: 2 data: [-0.28088596 1.04119928] [2.99245932]\n",
"idx: 3 data: [-1.41942608 -0.14294466] [-3.4857788]\n",
"idx: 4 data: [-0.94715851 -0.90926992] [-3.88762596]\n",
"idx: 5 data: [ 0.83067309 -1.60634229] [-2.1943678]\n",
"\n",
"### Data from val set ###\n",
"idx: 0 data: [1.34841145 0.66873659] [6.29068461]\n",
"idx: 1 data: [-0.27889547 1.60049314] [2.89593152]\n",
"idx: 0 data: [0.31728831 0.1915907 ] [0.61714501]\n",
"idx: 1 data: [0.19605744 0.15569143] [-0.31029128]\n",
"\n",
"### Data from test set ###\n",
"idx: 0 data: [1.7529717 0.62403886] [4.23077858]\n",
"idx: 1 data: [0.93764475 1.51392039] [6.6005128]\n",
"idx: 0 data: [ 0.60736842 -1.62492312] [-1.52729382]\n",
"idx: 1 data: [-0.33198421 0.91780232] [2.45557787]\n",
"\n",
"### Data from train set again ###\n",
"idx: 0 data: [-0.68057372 -0.94303927] [-4.48274392]\n",
"idx: 1 data: [-0.4473647 1.27208487] [1.70464728]\n",
"idx: 2 data: [-0.4682478 2.49858032] [6.32016286]\n",
"idx: 3 data: [-0.89608283 0.06252056] [0.42949332]\n",
"idx: 4 data: [-0.26576938 -0.77356473] [-2.50971628]\n",
"idx: 5 data: [1.51339474 0.83226233] [4.76266322]\n"
"idx: 0 data: [-0.03919358 -1.81524105] [-5.45285373]\n",
"idx: 1 data: [-0.33238759 -0.25239502] [-2.28257421]\n",
"idx: 2 data: [-0.28088596 1.04119928] [2.99245932]\n",
"idx: 3 data: [-1.41942608 -0.14294466] [-3.4857788]\n",
"idx: 4 data: [-0.94715851 -0.90926992] [-3.88762596]\n",
"idx: 5 data: [ 0.83067309 -1.60634229] [-2.1943678]\n"
]
}
],
Expand Down Expand Up @@ -702,8 +702,8 @@
{
"data": {
"text/plain": [
"array([[1.7529717 , 0.62403886],\n",
" [0.93764475, 1.51392039]])"
"array([[ 0.60736842, -1.62492312],\n",
" [-0.33198421, 0.91780232]])"
]
},
"execution_count": null,
Expand All @@ -727,8 +727,8 @@
{
"data": {
"text/plain": [
"array([[4.23077858],\n",
" [6.6005128 ]])"
"array([[-1.52729382],\n",
" [ 2.45557787]])"
]
},
"execution_count": null,
Expand Down Expand Up @@ -764,36 +764,36 @@
"length train: 4 length val: 2 length test: 2\n",
"\n",
"### Data from train set ###\n",
"idx: 0 data: [[-0.32379888 -0.78640497 -0.13864155]\n",
" [ 0.94517259 -0.41177372 -2.05279301]] [1.93366225]\n",
"idx: 1 data: [[ 0.94517259 -0.41177372 -2.05279301]\n",
" [-1.10574283 1.15062583 1.93366225]] [1.75030197]\n",
"idx: 2 data: [[-1.10574283 1.15062583 1.93366225]\n",
" [ 0.7707494 -0.75147112 1.75030197]] [-0.62698549]\n",
"idx: 3 data: [[ 0.7707494 -0.75147112 1.75030197]\n",
" [-1.85243524 -0.66748402 -0.62698549]] [-5.11745204]\n",
"idx: 0 data: [[ 1.31636985 0.7897913 -0.75019741]\n",
" [-1.0363052 -0.33563099 5.30310363]] [-1.73725544]\n",
"idx: 1 data: [[-1.0363052 -0.33563099 5.30310363]\n",
" [ 0.61748301 0.12942096 -1.73725544]] [0.75991095]\n",
"idx: 2 data: [[ 0.61748301 0.12942096 -1.73725544]\n",
" [ 0.97507757 0.60012032 0.75991095]] [3.60080094]\n",
"idx: 3 data: [[0.97507757 0.60012032 0.75991095]\n",
" [0.05424665 0.05414227 3.60080094]] [1.06903634]\n",
"\n",
"### Data from val set ###\n",
"idx: 0 data: [[-1.85243524 -0.66748402 -0.62698549]\n",
" [ 0.34430677 -0.55338817 -5.11745204]] [-1.61582716]\n",
"idx: 1 data: [[ 0.34430677 -0.55338817 -5.11745204]\n",
" [-0.81291444 0.86692756 -1.61582716]] [1.18482547]\n",
"idx: 0 data: [[ 0.05424665 0.05414227 3.60080094]\n",
" [-0.19670518 2.25039121 1.06903634]] [6.53359576]\n",
"idx: 1 data: [[-0.19670518 2.25039121 1.06903634]\n",
" [-1.84005742 -0.24281547 6.53359576]] [-3.96123686]\n",
"\n",
"### Data from test set ###\n",
"idx: 0 data: [[-0.81291444 0.86692756 -1.61582716]\n",
" [-0.75311363 1.67462759 1.18482547]] [2.34944632]\n",
"idx: 1 data: [[-0.75311363 1.67462759 1.18482547]\n",
" [-1.09058759 1.15451079 2.34944632]] [0.65357601]\n",
"idx: 0 data: [[-1.84005742 -0.24281547 6.53359576]\n",
" [ 0.53974671 1.48055778 -3.96123686]] [5.10164607]\n",
"idx: 1 data: [[ 0.53974671 1.48055778 -3.96123686]\n",
" [ 0.0885949 1.45853039 5.10164607]] [5.47333133]\n",
"\n",
"### Data from train set again ###\n",
"idx: 0 data: [[-0.32379888 -0.78640497 -0.13864155]\n",
" [ 0.94517259 -0.41177372 -2.05279301]] [1.93366225]\n",
"idx: 1 data: [[ 0.94517259 -0.41177372 -2.05279301]\n",
" [-1.10574283 1.15062583 1.93366225]] [1.75030197]\n",
"idx: 2 data: [[-1.10574283 1.15062583 1.93366225]\n",
" [ 0.7707494 -0.75147112 1.75030197]] [-0.62698549]\n",
"idx: 3 data: [[ 0.7707494 -0.75147112 1.75030197]\n",
" [-1.85243524 -0.66748402 -0.62698549]] [-5.11745204]\n"
"idx: 0 data: [[ 1.31636985 0.7897913 -0.75019741]\n",
" [-1.0363052 -0.33563099 5.30310363]] [-1.73725544]\n",
"idx: 1 data: [[-1.0363052 -0.33563099 5.30310363]\n",
" [ 0.61748301 0.12942096 -1.73725544]] [0.75991095]\n",
"idx: 2 data: [[ 0.61748301 0.12942096 -1.73725544]\n",
" [ 0.97507757 0.60012032 0.75991095]] [3.60080094]\n",
"idx: 3 data: [[0.97507757 0.60012032 0.75991095]\n",
" [0.05424665 0.05414227 3.60080094]] [1.06903634]\n"
]
}
],
Expand Down Expand Up @@ -1441,35 +1441,32 @@
" self.demand_lag_out_of_sample_test = self.demand_out_of_sample_test.copy()\n",
" self.demand_lag_out_of_sample_val = self.demand_out_of_sample_val.copy()\n",
"\n",
" print(np.sum(self.demand_lag_out_of_sample_val))\n",
" print(np.sum(self.demand_lag_out_of_sample_test))\n",
" print(np.sum(self.demand_out_of_sample_val))\n",
" print(np.sum(self.demand_out_of_sample_test))\n",
" \n",
"\n",
" # Normalize demand targets\n",
" if self.demand_normalization != 'no_normalization':\n",
" # Normalizing per SKU on time dimension\n",
" self.scaler_out_of_sample_test_demand.fit(self.demand_out_of_sample_test[:self.train_index_end+1])\n",
" transformed_demand = self.scaler_out_of_sample_test_demand.transform(self.demand_lag_out_of_sample_test)\n",
" self.demand_lag_out_of_sample_test.iloc[:,:] = transformed_demand\n",
" self.demand_out_of_sample_test.iloc[:,:] = transformed_demand\n",
"\n",
" self.scaler_out_of_sample_val_demand.fit(self.demand_out_of_sample_val[:self.train_index_end+1])\n",
" transformed_demand = self.scaler_out_of_sample_val_demand.transform(self.demand_lag_out_of_sample_val)\n",
" self.demand_lag_out_of_sample_val.iloc[:,:] = transformed_demand\n",
" self.demand_out_of_sample_val.iloc[:,:] = transformed_demand\n",
" \n",
" # Set unit size for demand targets\n",
" if self.demand_unit_size != None:\n",
" self.demand_lag_out_of_sample_test = np.round(self.demand_lag_out_of_sample_test, self.demand_unit_size)\n",
" self.demand_lag_out_of_sample_val = np.round(self.demand_lag_out_of_sample_val, self.demand_unit_size)\n",
" self.demand_out_of_sample_test = np.round(self.demand_out_of_sample_test, self.demand_unit_size)\n",
" self.demand_out_of_sample_val = np.round(self.demand_out_of_sample_val, self.demand_unit_size)\n",
"\n",
" # If separate normalization for lag demand, normalize it\n",
" if self.lag_demand_normalization != self.demand_normalization:\n",
" if self.lag_demand_normalization != 'no_normalization':\n",
" \n",
" self.demand_lag_out_of_sample_test = self.demand_out_of_sample_test.copy()\n",
" self.scaler_out_of_sample_test_demand_lag.fit(self.demand_lag_out_of_sample_test[:self.train_index_end+1])\n",
" transformed_demand_lag = self.scaler_out_of_sample_test_demand_lag.transform(self.demand_lag_out_of_sample_test)\n",
" self.demand_lag_out_of_sample_test.iloc[:,:] = transformed_demand_lag\n",
" \n",
" self.demand_lag_out_of_sample_val = self.demand_out_of_sample_val.copy()\n",
" self.scaler_out_of_sample_val_demand_lag.fit(self.demand_lag_out_of_sample_val[:self.train_index_end+1])\n",
" transformed_demand_lag = self.scaler_out_of_sample_val_demand_lag.transform(self.demand_lag_out_of_sample_val)\n",
" self.demand_lag_out_of_sample_val.iloc[:,:] = transformed_demand_lag\n",
Expand All @@ -1479,7 +1476,6 @@
" self.demand_lag_out_of_sample_test = self.demand_out_of_sample_test.copy()\n",
" self.demand_lag_out_of_sample_val = self.demand_out_of_sample_val.copy()\n",
" \n",
" \n",
" if self.SKU_features is not None:\n",
" logging.info(\"--Normalizing SKU features\")\n",
" # Normalizing across SKUs, no time dimension present\n",
Expand Down Expand Up @@ -1516,6 +1512,12 @@
" \n",
" self.normalized_out_of_sample_SKUs = True\n",
"\n",
" print(self.demand_out_of_sample_test)\n",
" print(self.demand_out_of_sample_val)\n",
"\n",
" print(self.demand_lag_out_of_sample_test)\n",
" print(self.demand_lag_out_of_sample_val)\n",
"\n",
" else:\n",
" raise NotImplementedError('Training data can only normalized during initialization - later normlization not implemented yet')\n",
"\n",
Expand Down
Loading