Commit 364915c8 authored by moebius/ALUG's avatar moebius/ALUG Committed by Christopher Spinrath
Browse files

Mass Storage Cloner for writing many USB sticks - 1st version

parent 042ff5ad
Mass Storage Cloner
(C)2014-2015 Martin Suefke
License: GPLv3 or newer, the GNU Public Licence, see http://www.gnu.org/licenses/gpl.html
Use this tool on a linux machine to write any image file to a target device selected in a GUI.
Prerequisites:
python 2.6 + Tk
xterm
pv
dd
Select the source file by editing 'write.sh'
Start the program with 'python gui.py'
First release 2015-03-26, version history in 'gui.py'
# -*- coding: utf-8 -*-
"""
Storage Abstraction in Python Classes
(C) 2014 Martin Süfke
License: GPLv3 or newer: http://www.gnu.org/licenses/gpl.html
"""
# Proper logging
from pprint import pformat
import logging
if __name__ == "__main__":
#logging.basicConfig(level=logging.WARN)
raise Exception("{0} cannot run as the main python program".format(__file__))
if len(logging.root.handlers) == 0:
# SPE compatible logging format
logging.basicConfig(level=logging.DEBUG,format='%(levelno)2d:%(funcName)s:%(message)s:File "%(filename)s", line %(lineno)d')
logger = logging.getLogger(__name__)
import inspect
from dbus import String as _dbus_String
from dbus import Array as _dbus_Array
from dbus import DBusException
def Implement(msg=""):
logger.warn('Implement File "%s", line %s, %s'+msg,*inspect.stack()[1][1:4])
def DBusStringArrayToString(Array_Or_String):
if isinstance(Array_Or_String, _dbus_Array):
return ''.join(chr(c) for c in Array_Or_String if c <> 0).strip()
elif isinstance(Array_Or_String, str):
return Array_Or_String.rstrip('\x00').strip()
class Storage(object):
""" Represents system data storage
may have multiple StorageDrives
"""
def __init__(self):
self._ChildClass = StorageDrive
self.StorageDrives = {}
def AddDrive(self, UDisks2Path, UDisks2Dict, EjectFunc, PowerOffFunc):
if UDisks2Dict is None:
logger.error("AddDrive() Empty dictionary for %s",UDisks2Path)
return
self.StorageDrives[UDisks2Path] = self._ChildClass(self, UDisks2Path, UDisks2Dict, EjectFunc, PowerOffFunc)
def RemoveDrive(self, UDisks2Path):
Drive = self.StorageDrives.pop(UDisks2Path, None)
if Drive is not None:
Drive._close()
else:
logger.warn("No Drive by Path %s", UDisks2Path)
def AddBlockDevice(self, UDisks2Path, UDisks2Dict):
logger.debug('Storage U2d.k: %s', UDisks2Dict.keys())
drive = UDisks2Dict[_dbus_String(u'Drive')]
self.StorageDrives[drive].AddBlockDevice(UDisks2Path, UDisks2Dict)
def RemoveBlockDevice(self, UDisks2Path):
BlockDev = self.GetStorageBlockDeviceByPath(UDisks2Path)
if BlockDev is not None:
Drive = BlockDev.Parent
Drive.RemoveBlockDevice(UDisks2Path)
else:
logger.warn("No BlockDevice by Path %s", UDisks2Path)
def AddFileSystem(self, UDisks2Path, UDisks2Dict, MountFunc, UnmountFunc):
if UDisks2Dict is None:
logger.info("AddFileSystem() Empty dictionary for %s",UDisks2Path)
return
#logger.debug("UDisks2Path = %s, Dict = %s)",UDisks2Path, UDisks2Dict)
BlockDev = self.GetStorageBlockDeviceByPath(UDisks2Path)
if BlockDev is None:
logger.warn("AddFileSystem( UDisks2Path = %s, Dict = %s) No BlockDevice",UDisks2Path, UDisks2Dict)
else:
BlockDev.AddFileSystem(UDisks2Path, UDisks2Dict, MountFunc, UnmountFunc)
def RemoveFileSystem(self, UDisks2Path):
BlockDev = self.GetStorageBlockDeviceByPath(UDisks2Path)
if BlockDev is not None:
BlockDev.RemoveFileSystem(UDisks2Path)
else:
logger.warn("No BlockDevice for FileSytem Path %s", UDisks2Path)
def UpdateMountPoints(self, UDisks2Path, UDisk2Mountpoints, UnmountFunc):
"""Update all Filesystems matched by UDisks2Path to reflect (new) UDisk2Mountpoints"""
BlockDev = self.GetStorageBlockDeviceByPath(UDisks2Path)
if BlockDev is None:
logger.warn("UDisks2Path %s: No such BlockDevice",UDisks2Path)
else:
BlockDev.UpdateMountPoints(UDisks2Path, UDisk2Mountpoints, UnmountFunc)
def GetStorageBlockDeviceByPath(self, aUDisks2Path):
""" Walk through all StorageDrives and look for a StorageBlockDevice with given UDisks2Path
Returns the BlockDevice
"""
for drive in self.StorageDrives.itervalues():
blkdev = drive.GetStorageBlockDeviceByPath(aUDisks2Path)
if blkdev is not None:
return blkdev
return None
class StorageDrive(object):
""" Represents a physical drive/device/medium
parent is Storage
may have multiple StorageBlockDevices
can Eject() -> will not detect drive any more until physical (?) reconnect
"""
def __init__(self, Parent, UDisks2Path, UDisks2Dict, EjectFunc, PowerOffFunc):
self._ChildClass = StorageBlockDevice
self.Parent = Parent
self.UDisks2Path = UDisks2Path
self.UDisks2Dict = UDisks2Dict
self.EjectFunc = EjectFunc
self.PowerOffFunc = PowerOffFunc
for name in "Id Size".split():
setattr(self, name, UDisks2Dict.get(_dbus_String(unicode(name))))
self.BlockDevices = {}
def _close(self):
""" a manual destructor """
pass
def AddBlockDevice(self, UDisks2Path, UDisks2Dict):
if UDisks2Dict is None:
logger.error("AddBlockDevice() Empty dictionary for %s",UDisks2Path)
return
#logger.debug("AddBlockDevice( Path = %s, Dict.Keys = %s)", UDisks2Path, UDisks2Dict.keys())
blockdev = self._ChildClass(self, UDisks2Path, UDisks2Dict)
self.BlockDevices[UDisks2Path] = blockdev
def RemoveBlockDevice(self, UDisks2Path):
BlkDev = self.BlockDevices.pop(UDisks2Path, None)
if BlkDev is not None:
BlkDev._close()
else:
logger.warn("No BlockDevice by Path %s", UDisks2Path)
def GetStorageBlockDeviceByPath(self, aUDisks2Path):
""" Get StorageBlockDevice by UDisks2Path """
return self.BlockDevices.get(aUDisks2Path)
def Eject(self):
logger.debug("StorageDrive.Eject() on %s",self.UDisks2Path)
if callable(self.EjectFunc):
self.EjectFunc(self.UDisks2Path)
else:
logger.error("StorageDrive.Eject has uncallable EjectFunc")
def PowerOff(self):
logger.debug("StorageDrive.Poweroff() on %s",self.UDisks2Path)
if callable(self.PowerOffFunc):
self.PowerOffFunc(self.UDisks2Path)
else:
logger.error("StorageDrive.PowerOff has uncallable PowerOffFunc")
def __str__(self):
return '{0:s}( UDisks2Path = "{1:s}")'.format(self.__class__.__name__, self.UDisks2Path)
class StorageBlockDevice(object):
""" Represents a block device '/dev/sdXYZ'
parent is StorageDrive
may have a single StorageFileSystem
can ReScan() -> does nothing (?)
"""
def __init__(self, Parent, UDisks2Path, UDisks2Dict=None, ChildClass=None):
assert UDisks2Dict is not None, "StorageBlockDevice( UDisks2Path = {0:s} ), UDisks2Dict is None. ".format(UDisks2Path)
#Implement(pformat([UDisks2Path, UDisks2Dict.keys()]))
#logger.debug("StorageBlockDevice.__init__(): ChildClass %s", ChildClass.__name__)
self._ChildClass = ChildClass or StorageFileSystem
self.Parent = Parent
self.UDisks2Path = UDisks2Path
self.UDisks2Dict = UDisks2Dict
self.Size = UDisks2Dict.get(_dbus_String(u'Size'))
logger.debug('Getting PreferredDevice')
self.PreferredDevice = DBusStringArrayToString(UDisks2Dict.get(_dbus_String(u'PreferredDevice'),''))
# Filesystem properties: IdUUID + IdLabel , IdUsage + IdType + IdVersion
# Crude hack: if IdUsage == Filesystem > "" then create a FS object
# Even worse Hack for ease of use:
# Leave Filesytem alone for now, add it when AddFileSystem() is called
self.FileSystem = None
self.FsParms = dict() # keep these
for name in "IdUUID IdLabel IdUsage IdType IdVersion".split():
self.FsParms[name] = str(UDisks2Dict.get(_dbus_String(unicode(name))).strip('\x00').strip())
# if len(self.FsParms['IdUsage'])>0:
# logger.info("got Filesystem info, not adding yet: %s using Class %s",
# pformat(self.FsParms),
# self._ChildClass.__name__
# )
def _close(self):
""" a manual destructor """
pass
def AddFileSystem(self, UDisks2Path, UDisks2Dict, MountFunc, UnmountFunc):
""" Add the info in the "Filesystem" dictionary or UDisks2.
That means the mount points, basically
"""
if self.FileSystem is None:
self.FileSystem = self._ChildClass(self, MountFunc, **self.FsParms)
#logger.warn("Trying to AddFileSystemPoint() to StorageBlockDevice %s which has no (known) Filesystem",self.UDisks2Path)
#else:
self.FileSystem.AddMountPoints(UDisks2Path, UDisks2Dict, UnmountFunc)
def RemoveFileSystem(self, UDisks2Path):
if UDisks2Path == self.UDisks2Path:
if self.FileSystem is not None:
fs=self.FileSystem
self.FileSystem = None
fs._close()
else:
logger.warn("Trying to remove non-existent filesystem by Path %s", UDisks2Path)
else:
logger.warn("No FileSystem by Path %s", UDisks2Path)
def UpdateMountPoints(self, UDisks2Path, UDisk2Mountpoints, UnmountFunc):
"""Update the BlockDevice's FileSystem to reflect (new) UDisk2Mountpoints"""
assert self.UDisks2Path == UDisks2Path, "UpdateMountPoints() with wrong UDisks2Path, expected {0!s}, got {1!s}".format(self.UDisks2Path, UDisks2Path)
assert self.FileSystem is not None, "UpdateMountPoints() FileSystem is None"
self.FileSystem.UpdateMountPoints(UDisks2Path, UDisk2Mountpoints, UnmountFunc)
def ReScan(self):
Implement()
def __str__(self):
return '{0:s}( UDisks2Path = "{1:s}")'.format(self.__class__.__name__, self.UDisks2Path)
class StorageFileSystem(object):
""" Represents a Filesystem (vfat, ext, ...)
parent is BlockDevice
may have multiply StorageMountpoint
can Mount()
"""
def __init__(self, Parent, MountFunc, IdUUID, IdLabel, IdUsage, IdType, IdVersion):
self._ChildClass = StorageMountPoint
self.Parent = Parent
self.MountFunc = MountFunc # Call this to mount the filesystem
self.IdUUID = IdUUID
self.IdLabel = IdLabel
self.IdUsage = IdUsage
self.IdType = IdType
self.IdVersion = IdVersion
self.MountPoints = {}
def _close(self):
""" a manual destructor """
pass
def AddMountPoints(self, UDisks2Path, UDisks2Dict, UnmountFunc):
""" Add all MountPoints based on the UDisks2 Filesystem dictionary """
if UDisks2Path <> self.Parent.UDisks2Path:
logger.error("AddMountPoint() UDisks2Path given >%s< differs from Parent.Path >%s<.", UDisks2Path, self.Parent.UDisks2Path)
return
U2MountPoints = UDisks2Dict.get(_dbus_String(u'MountPoints'), {})
for U2MPoint in U2MountPoints:
MountPoint = ''.join(chr(c) for c in U2MPoint if c <> 0).strip()
#logger.debug("got Mountpoints %s", MountPoint)
self.AddMountPoint(UDisks2Path, MountPoint, UnmountFunc)
def AddMountPoint(self, UDisks2Path, MountPoint, UnmountFunc):
""" Add single MountPoint based on the UDisks2Path and the MountPoint directory """
MountPointObj = self._ChildClass(self, UDisks2Path, MountPoint, UnmountFunc)
self.MountPoints[MountPoint]=MountPointObj
def UpdateMountPoints(self, UDisks2Path, UDisk2Mountpoints, UnmountFunc):
"""Update this FileSystem to reflect (new) UDisk2Mountpoints"""
if UDisks2Path <> self.Parent.UDisks2Path:
logger.error("UpdateMountPoints() UDisks2Path given >%s< differs from Parent.Path >%s<.", UDisks2Path, self.Parent.UDisks2Path)
return
NewMountpoints=list(s.strip('\x00') for s in UDisk2Mountpoints)
DelMountpoints=list()
# First, kill all stale mountpoints
for oldmp in self.MountPoints: # keys are mount point paths
if oldmp in NewMountpoints:
logger.debug("known MP >%s<", oldmp)
NewMountpoints.remove(oldmp)
else:
logger.debug("stale MP to remove >%s<", oldmp)
DelMountpoints.append(oldmp)
for delmp in DelMountpoints:
logger.debug("delete stale MP >%s<", delmp)
self.MountPoints.pop(delmp)._close()
# Now create new mountpoints
for newmp in NewMountpoints:
logger.debug("create new MP >%s<", newmp)
self.AddMountPoint(UDisks2Path, newmp, UnmountFunc)
def __str__(self):
return '{0:s}( UDisks2Path = "{1:s}")'.format(self.__class__.__name__, self.Parent.UDisks2Path)
def Mount(self, *args, **kwargs):
logger.debug("StorageMountPoint.Mount args: %s %s",args, kwargs)
if callable(self.MountFunc):
try:
self.MountFunc(self.Parent.UDisks2Path)
except DBusException as dbe:
logger.warn(dbe.message)
else:
logger.error("StorageMountPoint.Mount has uncallable MountFunc")
class StorageMountPoint(object):
""" Represents a Filesystem (vfat, ext, ...)
parent is StorageFileSystem
can UnMount()
"""
def __init__(self, Parent, UDisks2Path, MountPoint, UnmountFunc):
self.Parent = Parent
self.UDisks2Path = UDisks2Path
self.MountPoint = MountPoint
self.UnmountFunc = UnmountFunc
#Implement()
# def __del__(self):
# logger.debug("%s.__del__()",self.__class__.__name__)
def _close(self):
""" A 'manual' destructor"""
logger.debug("%s _close for %s",self.__class__.__name__,str(self))
def __str__(self):
return '{0:s}( UDisks2Path = "{1:s}")'.format(self.__class__.__name__, self.UDisks2Path)
def Unmount(self, *args, **kwargs):
logger.debug("StorageMountPoint.Unmount args: %s %s", args, kwargs)
if callable(self.UnmountFunc):
try:
self.UnmountFunc(self.UDisks2Path)
except DBusException as dbe:
logger.warn(dbe.message)
else:
logger.error("StorageMountPoint.Unmount has uncallable UnmountFunc")
#end;
# -*- coding: utf-8 -*-
"""
TK Frames for Representation of Storage Abstraction
(C) 2014 Martin Süfke
License: GPLv3 or newer: http://www.gnu.org/licenses/gpl.html
"""
# Proper logging
from pprint import pprint, pformat
import logging
#import os # A dirty hack here
import subprocess
import shlex
import signal # future
if __name__ == "__main__":
#logging.basicConfig(level=logging.WARN)
raise Exception("{0} cannot run as the main python program".format(__file__))
if len(logging.root.handlers) == 0:
# SPE compatible logging format
logging.basicConfig(level=logging.DEBUG,format='%(levelno)2d:%(funcName)s:%(message)s:File "%(filename)s", line %(lineno)d')
logger = logging.getLogger(__name__)
import inspect
import StorageObjects
import Tkinter as Tk
#from Tkinter import BooleanVar, IntVar, DoubleVar, StringVar
#import tkMessageBox as dialog
#import tkFileDialog as filedialog
from Tkconstants import N, E, S, W, HORIZONTAL, LEFT, RIGHT, TOP, BOTTOM, \
BOTH, NORMAL, DISABLED, X, Y
def Implement(msg=""):
logger.warn('Implement File "%s", line %s, %s'+msg,*inspect.stack()[1][1:4])
def SizeStr(size):
""" Print size with thousands separators """
return "{0:,d} bytes".format(size)
class TkFStorage(StorageObjects.Storage):
""" Tk Frame for Storage """
def __init__(self, ParentFrame):
#logger.debug(self.__class__.__name__)
super(self.__class__,self).__init__()
self._ChildClass = TkFDrive
self.TKFrame = Tk.Frame(ParentFrame, borderwidth=2, relief='groove')
self.TKFrame.pack(side=TOP, fill=X)
#Tk.Label(self.TKFrame,text=self.__class__.__name__).pack(side=LEFT, fill=X)
def FilterViewDriveAdd(self, MoreFiltersAsDict):
""" Adds view filters on drives
example: FilterViewDriveAdd(self, {'Removable': True})
"""
#logger.debug("Num Drives %d", len(self.StorageDrives))
for k,v in self.StorageDrives.iteritems():
# logger.debug("\nDrive %s %s", k, str(v))
v.FilterViewAdd(MoreFiltersAsDict)
# TODO: Idea here is to retain the order of hidden/visible drives.
# That may not be possible in all circumstances
# Also, is may not be desirable. When showing devices, add them at the TOP
# f = v.TKFrame
# x = f.tk.call('pack', 'info', str(f))
## f._saved_w = f._w
## f._w = str(f)
## logger.debug("repr(f) %s",repr(f))
## logger.debug(" str(f) %s", str(f))
## x = f.pack_info()
## f._saved_w = f._w
## v.TKFrame.pack_forget()
# logger.debug("Pack_info: %s", x)
#
# logger.debug("Slave Frames %s", self.TKFrame.pack_slaves())
def FilterViewDriveRemove(self, FilterKeysAsList):
""" Removes view filters from drives
example: FilterViewDriveRemove(self, ['Removable', 'Ejectable']})
"""
for k,v in self.StorageDrives.iteritems():
v.FilterViewRemove(FilterKeysAsList)
class TkFDrive(StorageObjects.StorageDrive):
""" Tk Frame for Drive """
def __init__(self, *args, **kwargs):
"""
Pass argment to StorageObjects.StoragedDrive
FilterView = {} :
Dictionary of Key = Value pairs that make this entry invisible if found inside UDisks2Dict
"""
#logger.debug(self.__class__.__name__)
self.ViewFilter = kwargs.pop('FilterView', {})
super(self.__class__,self).__init__(*args, **kwargs)
self._ChildClass = TkFBlockdevice
self.TKFrame = Tk.Frame(self.Parent.TKFrame, borderwidth=2, relief='groove', background='light blue')
self.visible = False # Not pack()'ed yet
self.FilterView() # leads to self.TKFrame.Pack() if visible
Tk.Label(self.TKFrame,text="Drive", background='light blue').pack(side=LEFT, anchor="nw")
InfoFrame = Tk.Frame(self.TKFrame, borderwidth=2)
Tk.Button(InfoFrame,text="ignore", command = self.OnBtnIgnore, background='light blue').pack(side=RIGHT, anchor="e")
if self.UDisks2Dict.get('Ejectable',False):
Tk.Button(InfoFrame,text="Eject", command = self.OnBtnEject).pack(side=RIGHT, anchor="e")
if self.UDisks2Dict.get('CanPowerOff',False):
Tk.Button(InfoFrame,text="PowerOff", command = self.OnBtnPowerOff).pack(side=RIGHT, anchor="e")
Tk.Label( InfoFrame,text=self.Id).pack(anchor="nw")
# Size as a str ?!?
try:
Tk.Label( InfoFrame,text=SizeStr(self.Size)).pack(anchor="nw")
except ValueError:
logger.debug("self.Size: s:%s r:%s",str(self.Size), repr(self.Size))
InfoFrame.pack(side=TOP, fill=X, padx=5, pady=5)
def _close(self):
""" A manual destructor"""
self.TKFrame.pack_forget() # Remove visible component
self.TKFrame.destroy() # Kill visible component
self.TKFrame = None
super(self.__class__,self)._close()
def OnBtnEject(self):
self.Eject()
def OnBtnPowerOff(self):
self.PowerOff()
def OnBtnIgnore(self):
#self.Ignore = True # Make a Filter that always makes thisone ingnore itself.
self.FilterViewAdd({'Id':self.UDisks2Dict['Id']})
#logger.debug("UDisk2Dict: %s", pformat(self.UDisks2Dict))
#self.FilterMakeInvisible()
def FilterViewAdd(self, MoreFiltersAsDict):
""" Adds view filters on this drive
Filtered drives will not show themselves in the main gui.
example: FilterViewDriveAdd(self, {'Removable': True})
see also: FilterViewRemove, FilterView
"""
self.ViewFilter.update(MoreFiltersAsDict)
self.FilterView()
def FilterViewRemove(self, FilterKeysAsList):
""" Removes view filters from this drive
example: FilterViewDriveRemove(self, ['Removable', 'Ejectable']})
see also: FilterViewAdd, FilterView
"""
for k in FilterKeysAsList:
if k in self.ViewFilter:
del self.ViewFilter[k]
self.FilterView()
def FilterView(self):
""" Apply Filter against this drive
see also: FilterViewAdd, FilterViewRemove
"""
visible = True
for key, test in self.ViewFilter.iteritems():
# A little suprising, but self.UDisks2Dict[dbus.String(u'Foo')] is as
# good as self.UDisks2Dict['Foo']
#logger.debug("Testing key %s",key)
if self.UDisks2Dict.has_key(key):
#logger.debug("key found, value is %s", self.UDisks2Dict[key])
#logger.debug("Testing %s == %s", test, self.UDisks2Dict[key] == test)
if self.UDisks2Dict[key] == test:
visible = False
if visible:
self.FilterMakeVisible()
else:
self.FilterMakeInvisible()
def FilterMakeVisible(self):
if not self.visible:
#logger.debug("")
self.TKFrame.pack(side=BOTTOM, fill=X) # Pack on Bottom, so that new entries end up on top.
self.visible = True
def FilterMakeInvisible(self):
if self.visible:
#logger.debug("")
self.TKFrame.pack_forget()
self.visible = False
class TkFBlockdevice(StorageObjects.StorageBlockDevice):
""" Tk Frame for Blockdevice """
def __init__(self, *args, **kwargs):
#logger.debug(self.__class__.__name__)
#logger.debug("args %s kwargs %s", args, kwargs)
kwargs['ChildClass'] = TkFFilesystem
super(self.__class__,self).__init__(*args, **kwargs)
self.TKFrame = Tk.Frame(self.Parent.TKFrame, borderwidth=2, relief='groove', background='light green')
self.TKFrame.pack(side=BOTTOM, fill=X)
Tk.Label(self.TKFrame,text="BlkDev", background='light green').pack(side=LEFT, anchor="nw")
InfoFrame = Tk.Frame(self.TKFrame, borderwidth=2, relief='flat')
#Tk.Label(self.TKFrame,text=self.__class__.__name__).pack(side=TOP, fill=X)
Tk.Button(InfoFrame, text="Write", command=self.TkCmdWrite).pack(side=RIGHT)
Tk.Label( InfoFrame, text="{0:s}".format(self.PreferredDevice)).pack(anchor="nw")
Tk.Label( InfoFrame, text="Size "+SizeStr(self.Size)).pack(anchor="nw")
InfoFrame.pack(side=TOP, fill=X, padx=5, pady=5)
def _close(self):
""" A manual destructor"""
self.TKFrame.pack_forget() # Remove visible component
self.TKFrame.destroy() # Kill visible component
self.TKFrame = None
super(self.__class__,self)._close()
def TkCmdWrite(self):
logger.debug("Write to %s -> execute write.sh %s", self.PreferredDevice, self.PreferredDevice)
# TODO: This would need a nice subprocess.popen() ...
#os.system("./write.sh %s"%(self.PreferredDevice))
if True: #not self.proc:
cmdarray=" ".join( [
"./write.sh %s"%(self.PreferredDevice),
] )
args = shlex.split(cmdarray)
self.proc = subprocess.Popen(args,
#bufsize = 1, # line buffered
bufsize = 4096, # byte buffered
stdin = None,
stdout = None, #subprocess.PIPE,
stderr = None, #subprocess.STDOUT,
# preexec_fn unneeded
close_fds = True, # Do not leak file descriptors
#cwd = Current Working Directory
#env Environment mapping