Skip to content
This repository has been archived by the owner on Mar 30, 2023. It is now read-only.

Commit

Permalink
Merge pull request #48 from RSE-Cambridge/mount-failures
Browse files Browse the repository at this point in the history 
Mount failures
  • Loading branch information
@JohnGarbutt
JohnGarbutt authored Jan 7, 2019
2 parents 98ffcad + 3af7c86 commit 1a00af1
Show file tree
Hide file tree
Showing 7 changed files with 68 additions and 64 deletions.
4 changes: 2 additions & 2 deletions internal/pkg/lifecycle/brickmanager/bricks.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -227,7 +227,7 @@ func processAttach(poolRegistry registry.PoolRegistry, volumeRegistry registry.V
handleError(volumeRegistry, volume, err)
return
}
err = plugin.Mounter().Mount(volume, bricks) // TODO pass down specific attachments?
err = plugin.Mounter().Mount(volume, bricks, attachments) // TODO pass down specific attachments?
if err != nil {
handleError(volumeRegistry, volume, err)
return
Expand Down Expand Up @@ -260,7 +260,7 @@ func processDetach(poolRegistry registry.PoolRegistry, volumeRegistry registry.V
return
}

err = plugin.Mounter().Unmount(volume, bricks) // TODO pass down specific attachments?
err = plugin.Mounter().Unmount(volume, bricks, attachments) // TODO pass down specific attachments?
if err != nil {
// TODO: update specific attachment into an error state?
handleError(volumeRegistry, volume, err)
Expand Down
16 changes: 8 additions & 8 deletions internal/pkg/mocks/pfsprovider_mock.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

12 changes: 6 additions & 6 deletions internal/pkg/pfsprovider/ansible/mount.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ func getMdtSize() string {
return mdtSize
}

func mount(fsType FSType, volume registry.Volume, brickAllocations []registry.BrickAllocation) error {
func mount(fsType FSType, volume registry.Volume, brickAllocations []registry.BrickAllocation, attachments []registry.Attachment) error {
log.Println("Mount for:", volume.Name)
var primaryBrickHost string
for _, allocation := range brickAllocations {
Expand All @@ -52,7 +52,7 @@ func mount(fsType FSType, volume registry.Volume, brickAllocations []registry.Br
executeAnsibleMount(fsType, volume, brickAllocations)
}

for _, attachment := range volume.Attachments {
for _, attachment := range attachments {
if attachment.State != registry.RequestAttach {
log.Printf("Skipping volume %s attach: %+v", volume.Name, attachment)
continue
Expand Down Expand Up @@ -118,10 +118,10 @@ func mount(fsType FSType, volume registry.Volume, brickAllocations []registry.Br
return nil
}

func umount(fsType FSType, volume registry.Volume, brickAllocations []registry.BrickAllocation) error {
func umount(fsType FSType, volume registry.Volume, brickAllocations []registry.BrickAllocation, attachments []registry.Attachment) error {
log.Println("Umount for:", volume.Name)

for _, attachment := range volume.Attachments {
for _, attachment := range attachments {
if attachment.State != registry.RequestDetach {
log.Printf("Skipping volume %s detach for: %+v", volume.Name, attachment)
continue
Expand All @@ -133,10 +133,10 @@ func umount(fsType FSType, volume registry.Volume, brickAllocations []registry.B
swapFile := path.Join(mountDir, fmt.Sprintf("/swap/%s", attachment.Hostname)) // TODO share?
loopback := fmt.Sprintf("/dev/loop%d", volume.ClientPort) // TODO share?
if err := swapOff(attachment.Hostname, loopback); err != nil {
return err
log.Printf("Warn: failed to swap off %+v", attachment)
}
if err := detachLoopback(attachment.Hostname, loopback); err != nil {
return err
log.Printf("Warn: failed to detach loopback %+v", attachment)
}
if err := removeSubtree(attachment.Hostname, swapFile); err != nil {
return err
Expand Down
84 changes: 44 additions & 40 deletions internal/pkg/pfsprovider/ansible/mount_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,33 +97,34 @@ func Test_Mount(t *testing.T) {
defer func() { runner = &run{} }()
fake := &fakeRunner{}
runner = fake
attachments := []registry.Attachment{
{Hostname: "client1", Job: "job1", State: registry.RequestAttach},
{Hostname: "client2", Job: "job1", State: registry.RequestAttach},
{Hostname: "client3", Job: "job3", State: registry.Attached},
{Hostname: "client3", Job: "job3", State: registry.RequestDetach},
{Hostname: "client3", Job: "job3", State: registry.Detached},
{Hostname: "client2", Job: "job2", State: registry.RequestAttach},
}
volume := registry.Volume{
Name: "asdf", JobName: "asdf",
AttachGlobalNamespace: true,
AttachPrivateNamespace: true,
AttachAsSwapBytes: 1024 * 1024, // 1 MiB
Attachments: []registry.Attachment{
{Hostname: "client1", Job: "job1", State: registry.RequestAttach},
{Hostname: "client2", Job: "job1", State: registry.RequestAttach},
{Hostname: "client3", Job: "job3", State: registry.Attached},
{Hostname: "client3", Job: "job3", State: registry.RequestDetach},
{Hostname: "client3", Job: "job3", State: registry.Detached},
{Hostname: "client2", Job: "job2", State: registry.RequestAttach},
},
ClientPort: 42,
Owner: 1001,
Group: 1001,
Attachments: attachments,
ClientPort: 42,
Owner: 1001,
Group: 1001,
}

assert.PanicsWithValue(t,
"failed to find primary brick for volume: asdf",
func() { mount(Lustre, volume, nil) })
func() { mount(Lustre, volume, nil, nil) })

bricks := []registry.BrickAllocation{
{Hostname: "host1"},
{Hostname: "host2"},
}
err := mount(Lustre, volume, bricks)
err := mount(Lustre, volume, bricks, attachments)
assert.Nil(t, err)
assert.Equal(t, 53, fake.calls)

Expand Down Expand Up @@ -162,28 +163,29 @@ func Test_Umount(t *testing.T) {
defer func() { runner = &run{} }()
fake := &fakeRunner{}
runner = fake
attachments := []registry.Attachment{
{Hostname: "client1", Job: "job4", State: registry.RequestDetach},
{Hostname: "client2", Job: "job4", State: registry.RequestDetach},
{Hostname: "client3", Job: "job3", State: registry.Attached},
{Hostname: "client3", Job: "job3", State: registry.RequestAttach},
{Hostname: "client3", Job: "job3", State: registry.Detached},
{Hostname: "client2", Job: "job1", State: registry.RequestDetach},
}
volume := registry.Volume{
Name: "asdf", JobName: "asdf",
AttachGlobalNamespace: true,
AttachPrivateNamespace: true,
AttachAsSwapBytes: 10000,
Attachments: []registry.Attachment{
{Hostname: "client1", Job: "job4", State: registry.RequestDetach},
{Hostname: "client2", Job: "job4", State: registry.RequestDetach},
{Hostname: "client3", Job: "job3", State: registry.Attached},
{Hostname: "client3", Job: "job3", State: registry.RequestAttach},
{Hostname: "client3", Job: "job3", State: registry.Detached},
{Hostname: "client2", Job: "job1", State: registry.RequestDetach},
},
ClientPort: 42,
Owner: 1001,
Group: 1001,
Attachments: attachments,
ClientPort: 42,
Owner: 1001,
Group: 1001,
}
bricks := []registry.BrickAllocation{
{Hostname: "host1"},
{Hostname: "host2"},
}
err := umount(Lustre, volume, bricks)
err := umount(Lustre, volume, bricks, attachments)
assert.Nil(t, err)
assert.Equal(t, 20, fake.calls)

Expand All @@ -207,24 +209,25 @@ func Test_Umount_multi(t *testing.T) {
defer func() { runner = &run{} }()
fake := &fakeRunner{}
runner = fake
attachments := []registry.Attachment{
{Hostname: "client1", Job: "job1", State: registry.RequestDetach},
}
volume := registry.Volume{
MultiJob: true,
Name: "asdf", JobName: "asdf",
AttachGlobalNamespace: true,
AttachPrivateNamespace: true,
AttachAsSwapBytes: 10000,
Attachments: []registry.Attachment{
{Hostname: "client1", Job: "job1", State: registry.RequestDetach},
},
ClientPort: 42,
Owner: 1001,
Group: 1001,
Attachments: attachments,
ClientPort: 42,
Owner: 1001,
Group: 1001,
}
bricks := []registry.BrickAllocation{
{Hostname: "host1"},
{Hostname: "host2"},
}
err := umount(Lustre, volume, bricks)
err := umount(Lustre, volume, bricks, attachments)
assert.Nil(t, err)
assert.Equal(t, 3, fake.calls)

Expand All @@ -238,25 +241,26 @@ func Test_Mount_multi(t *testing.T) {
defer func() { runner = &run{} }()
fake := &fakeRunner{}
runner = fake
attachments := []registry.Attachment{
{Hostname: "client1", Job: "job1", State: registry.RequestAttach},
}
volume := registry.Volume{
MultiJob: true,
Name: "asdf", JobName: "asdf",
AttachGlobalNamespace: true,
AttachPrivateNamespace: true,
AttachAsSwapBytes: 10000,
Attachments: []registry.Attachment{
{Hostname: "client1", Job: "job1", State: registry.RequestAttach},
},
ClientPort: 42,
Owner: 1001,
Group: 1001,
UUID: "medkDfdg",
Attachments: attachments,
ClientPort: 42,
Owner: 1001,
Group: 1001,
UUID: "medkDfdg",
}
bricks := []registry.BrickAllocation{
{Hostname: "host1"},
{Hostname: "host2"},
}
err := mount(Lustre, volume, bricks)
err := mount(Lustre, volume, bricks, attachments)
assert.Nil(t, err)
assert.Equal(t, 5, fake.calls)

Expand Down
8 changes: 4 additions & 4 deletions internal/pkg/pfsprovider/ansible/plugin.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,10 +87,10 @@ type mounter struct {
FSType FSType
}

func (mounter *mounter) Mount(volume registry.Volume, brickAllocations []registry.BrickAllocation) error {
return mount(mounter.FSType, volume, brickAllocations)
func (mounter *mounter) Mount(volume registry.Volume, brickAllocations []registry.BrickAllocation, attachments []registry.Attachment) error {
return mount(mounter.FSType, volume, brickAllocations, attachments)
}

func (mounter *mounter) Unmount(volume registry.Volume, brickAllocations []registry.BrickAllocation) error {
return umount(mounter.FSType, volume, brickAllocations)
func (mounter *mounter) Unmount(volume registry.Volume, brickAllocations []registry.BrickAllocation, attachments []registry.Attachment) error {
return umount(mounter.FSType, volume, brickAllocations, attachments)
}
4 changes: 2 additions & 2 deletions internal/pkg/pfsprovider/fake/plugin.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,12 +44,12 @@ func (*volumeProvider) CopyDataOut(volume registry.Volume) error {

type mounter struct{}

func (*mounter) Mount(volume registry.Volume, brickAllocations []registry.BrickAllocation) error {
func (*mounter) Mount(volume registry.Volume, brickAllocations []registry.BrickAllocation, attachments []registry.Attachment) error {
log.Println("Mount for:", volume.Name)
return nil
}

func (*mounter) Unmount(volume registry.Volume, brickAllocations []registry.BrickAllocation) error {
func (*mounter) Unmount(volume registry.Volume, brickAllocations []registry.BrickAllocation, attachments []registry.Attachment) error {
log.Println("Umount for:", volume.Name)
return nil
}
4 changes: 2 additions & 2 deletions internal/pkg/pfsprovider/interface.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,6 @@ type VolumeProvider interface {
// Actions that are sent to remote hosts,
// typically compute nodes and primary brick hosts
type Mounter interface {
Mount(volume registry.Volume, brickAllocations []registry.BrickAllocation) error
Unmount(volume registry.Volume, brickAllocations []registry.BrickAllocation) error
Mount(volume registry.Volume, brickAllocations []registry.BrickAllocation, attachments []registry.Attachment) error
Unmount(volume registry.Volume, brickAllocations []registry.BrickAllocation, attachments []registry.Attachment) error
}

0 comments on commit 1a00af1

Please sign in to comment.