Skip to content

Commit

Permalink
Fix params_test on Windows. (#7)
Browse files Browse the repository at this point in the history
Fix was similar to
iree-org/iree@11d2259

Once a temporary file is opened on Windows, it can't be reopened unless
some extra conditions are met. From
https://docs.python.org/3/library/tempfile.html:

> Opening the temporary file again by its name while it is still open
works as follows:
> * On POSIX the file can always be opened again.
> * On Windows, make sure that at least one of the following conditions
are fulfilled:
>     * _delete_ is false
> * additional open shares delete access (e.g. by calling
[os.open()](https://docs.python.org/3/library/os.html#os.open) with the
flag O_TEMPORARY)
> * delete is true but delete_on_close is false. Note, that in this case
the additional opens that do not share delete access (e.g. created via
builtin [open()](https://docs.python.org/3/library/functions.html#open))
must be closed before exiting the context manager, else the
[os.unlink()](https://docs.python.org/3/library/os.html#os.unlink) call
on context manager exit will fail with a
[PermissionError](https://docs.python.org/3/library/exceptions.html#PermissionError).

We were setting `delete=False`, but I still saw these errors:

```
__________________________________ ParamsTest.testCreateArchive ___________________________________

self = <params_test.ParamsTest testMethod=testCreateArchive>

    def testCreateArchive(self):
        with tempfile.NamedTemporaryFile("wb", delete=False, suffix=".irpa") as f:
            file_path = Path(f.name)
            try:
                m = SimpleParamsModule()
>               save_module_parameters(file_path, m)

tests\aot\params_test.py:43:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ shark_turbine\aot\params.py:136: in save_module_parameters
    builder.save(file_path)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

self = <shark_turbine.aot.params.ParameterArchiveBuilder object at 0x00000216279F0A50>
file_path = WindowsPath('C:/Users/Scott/AppData/Local/Temp/tmpj0_157nn.irpa')

    def save(self, file_path: Union[str, Path]):
        """Saves the archive."""
>       self._index.create_archive_file(str(file_path))
E       RuntimeError: Error building parameter archive: D:\a\iree\iree\c\runtime\src\iree\base\internal\file_io.c:381: UNKNOWN; failed to open file 'C:\Users\Scott\AppData\Local\Temp\tmpj0_157nn.irpa'

shark_turbine\aot\params.py:261: RuntimeError
```
  • Loading branch information
ScottTodd authored Apr 30, 2024
1 parent 3520958 commit 63a2411
Showing 1 changed file with 24 additions and 30 deletions.
54 changes: 24 additions & 30 deletions tests/aot/params_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,38 +36,32 @@ def forward(self, x):

class ParamsTest(unittest.TestCase):
def testCreateArchive(self):
with tempfile.NamedTemporaryFile("wb", delete=False, suffix=".irpa") as f:
file_path = Path(f.name)
try:
m = SimpleParamsModule()
save_module_parameters(file_path, m)
# mmap=False is a bit nicer for tests on Windows because it doesn't
# lock the file for an arbitrary duration.
archive = ParameterArchive(file_path, mmap=False)
items = dict(archive.items())
weight = items["classifier.weight"].as_tensor()
bias = items["classifier.bias"].as_tensor()
torch.testing.assert_close(weight, m.classifier.weight)
torch.testing.assert_close(bias, m.classifier.bias)
finally:
file_path.unlink()
with tempfile.TemporaryDirectory() as td:
file_path = Path(td) / "archive.irpa"
m = SimpleParamsModule()
save_module_parameters(file_path, m)
# mmap=False is a bit nicer for tests on Windows because it doesn't
# lock the file for an arbitrary duration.
archive = ParameterArchive(file_path, mmap=False)
items = dict(archive.items())
weight = items["classifier.weight"].as_tensor()
bias = items["classifier.bias"].as_tensor()
torch.testing.assert_close(weight, m.classifier.weight)
torch.testing.assert_close(bias, m.classifier.bias)

def testCreateArchiveWithPrefixScope(self):
with tempfile.NamedTemporaryFile("wb", delete=False, suffix=".irpa") as f:
file_path = Path(f.name)
try:
m = SimpleParamsModule()
save_module_parameters(file_path, m, prefix="foobar.model")
# mmap=False is a bit nicer for tests on Windows because it doesn't
# lock the file for an arbitrary duration.
archive = ParameterArchive(file_path, mmap=False)
items = dict(archive.items())
weight = items["foobar.model.classifier.weight"].as_tensor()
bias = items["foobar.model.classifier.bias"].as_tensor()
torch.testing.assert_close(weight, m.classifier.weight)
torch.testing.assert_close(bias, m.classifier.bias)
finally:
file_path.unlink()
with tempfile.TemporaryDirectory() as td:
file_path = Path(td) / "archive.irpa"
m = SimpleParamsModule()
save_module_parameters(file_path, m, prefix="foobar.model")
# mmap=False is a bit nicer for tests on Windows because it doesn't
# lock the file for an arbitrary duration.
archive = ParameterArchive(file_path, mmap=False)
items = dict(archive.items())
weight = items["foobar.model.classifier.weight"].as_tensor()
bias = items["foobar.model.classifier.bias"].as_tensor()
torch.testing.assert_close(weight, m.classifier.weight)
torch.testing.assert_close(bias, m.classifier.bias)

def testExportExternalized(self):
m = SimpleParamsModule()
Expand Down

0 comments on commit 63a2411

Please sign in to comment.