diff --git a/.gitignore b/.gitignore index 315ab6f..516c850 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ __pycache__/ *# dist/ build/ +build_fs/ *.spec venv diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..5587414 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "iQuailFileServer"] + path = iQuailFileServer + url = https://github.com/QuailTeam/iQuailFileServer diff --git a/examples/side_img.gif b/examples/side_img.gif index 11fdbf9..c45ad5b 100644 Binary files a/examples/side_img.gif and b/examples/side_img.gif differ diff --git a/examples/test_fileserver.py b/examples/test_fileserver.py new file mode 100755 index 0000000..b7f82eb --- /dev/null +++ b/examples/test_fileserver.py @@ -0,0 +1,40 @@ +#!/usr/bin/python3 + +import iquail + +if iquail.helper.OS_LINUX: + iquail.run( + solution=iquail.SolutionFileServer('localhost', '4242', + './build_fs', '../iQuailFileServer'), + #solution=iquail.SolutionFileServer('localhost', '4242', '../iQuailFileServer/build'), + installer=iquail.Installer( + name='Allum1', + publisher='alies', + icon='icon.jpg', + binary='allum1', + console=True, + launch_with_quail=True + ), + builder=iquail.builder.Builder(), + controller=iquail.ControllerTkinter() + ) +else: + iquail.run( + solution=iquail.SolutionFileServer('192.168.0.13', '4242', '../iQuailFileServer/build'), + installer=iquail.Installer( + publisher="Michael Moller", + name='OpenHardwareMonitor', + icon='OpenHardwareMonitor.exe', + binary='OpenHardwareMonitor.exe', + console=True, + launch_with_quail=True, + requires_root=True + ), + builder=iquail.builder.Builder( + iquail.builder.CmdIcon('icon.ico'), + iquail.builder.CmdNoconsole(), + side_img_override="side_img.gif", + ), + controller=iquail.ControllerTkinter() + ) + diff --git a/examples/test_fileserver_xonotic.py b/examples/test_fileserver_xonotic.py new file mode 100755 index 0000000..434dc63 --- /dev/null +++ b/examples/test_fileserver_xonotic.py @@ -0,0 +1,21 @@ +#!/usr/bin/python3 + +import iquail + +if not iquail.helper.OS_LINUX: + raise AssertionError("This test solution is linux only") + +iquail.run( + solution=iquail.SolutionFileServer('localhost', '4242', + './build_fs', '../iQuailFileServer'), + installer=iquail.Installer( + name='Xonotic', + publisher='OHM', + icon='misc/logos/icons_png/xonotic_512.png', + binary='xonotic-linux64-sdl', + console=True, + launch_with_quail=True + ), + builder=iquail.builder.Builder(side_img_override='side_img.gif'), + controller=iquail.ControllerTkinter() +) diff --git a/examples/xonotic.py b/examples/xonotic.py new file mode 100755 index 0000000..0bf4b18 --- /dev/null +++ b/examples/xonotic.py @@ -0,0 +1,31 @@ +#!/usr/bin/python3 + +import iquail + +if iquail.helper.OS_LINUX: + binary = "xonotic-linux64-sdl" + icon = "misc/logos/icons_png/xonotic_512.png" + +if iquail.helper.OS_WINDOWS: + binary = "xonotic-x86.exe" + icon = "xonotic-x86.exe" + +iquail.run( + solution=iquail.SolutionPacked(path='Xonotic'), + installer=iquail.Installer( + publisher="OHM", + name='Xonotic', + icon=icon, + binary=binary, + console=False, + launch_with_quail=False, + is_large_solution=True, + ), + + # iquail.builder.CmdIcon('icon.ico'), + builder=iquail.builder.Builder( + iquail.builder.CmdNoconsole(), + side_img_override='side_img.gif', + ), + controller=iquail.ControllerTkinter() +) diff --git a/iQuailFileServer b/iQuailFileServer new file mode 160000 index 0000000..386c840 --- /dev/null +++ b/iQuailFileServer @@ -0,0 +1 @@ +Subproject commit 386c840c0d36c79e4647960b0af8343bf1c5cea7 diff --git a/iquail/builder/__init__.py b/iquail/builder/__init__.py index 070559f..81bca4f 100644 --- a/iquail/builder/__init__.py +++ b/iquail/builder/__init__.py @@ -5,3 +5,4 @@ from .cmd_icon import CmdIcon from .cmd_noconsole import CmdNoconsole from .cmd_integrity import CmdIntegrity +from .cmd_fileserver_client import CmdFileserverClient diff --git a/iquail/builder/cmd_fileserver_client.py b/iquail/builder/cmd_fileserver_client.py new file mode 100644 index 0000000..2dd5106 --- /dev/null +++ b/iquail/builder/cmd_fileserver_client.py @@ -0,0 +1,51 @@ +import os +import logging +from .. import helper +from .cmd_base import CmdBase +from ..errors import BuilderError + +logger = logging.getLogger(__name__) + + +class CmdFileserverClient(CmdBase): + """ Compile a Fileserver Client and add it to the executable + """ + + def __init__(self, build_path, fileserver_path, binary_name): + super().__init__() + self._binary_name = binary_name + self._fileserver_path = fileserver_path + if fileserver_path: # Build client + self._fileserver_path = os.path.abspath(self._fileserver_path) + # else: Client has already been built + self._build_path = build_path + self._build_path = os.path.abspath(self._build_path) + self._client_path = os.path.join(self._build_path, self._binary_name) + + def _run_cmake(self): + cmd = 'cmake -B' + self._build_path + cmd += ' -S' + self._fileserver_path + status = os.system(cmd) + if status != 0: + raise BuilderError('CmdFileserverClient: CMake failed') + + def _run_make(self): + cmd = 'make -j' + str(os.cpu_count()) + cmd += ' -C ' + self._build_path + cmd += ' ' + self._binary_name + status = os.system(cmd) + if status != 0: + raise BuilderError('CmdFileserverClient: Make failed') + + def pre_build(self): + if self._fileserver_path == None: + return + logger.info("Calling CMake:") + self._run_cmake() + logger.info("Calling Make:") + self._run_make() + + def get_build_params(self): + params = [] + params += ['--add-data', self._client_path + os.path.pathsep + '.'] + return params diff --git a/iquail/errors.py b/iquail/errors.py index de68d5d..8e68270 100644 --- a/iquail/errors.py +++ b/iquail/errors.py @@ -6,3 +6,19 @@ class SolutionUnreachableError(Exception): class SolutionNotRemovableError(Exception): """Raised if solution is not reachable""" pass + +class SolutionFileNotFoundError(SolutionUnreachableError): + """Raised if solution cannot retreive file""" + pass + +class SolutionVersionNotFoundError(SolutionUnreachableError): + """Raised if distant version cannot be selected""" + pass + +class SolutionDecompressionError(SolutionUnreachableError): + """Raised if distant version cannot be selected""" + pass + +class BuilderError(Exception): + """Raised if solution is not reachable""" + pass diff --git a/iquail/solution/__init__.py b/iquail/solution/__init__.py index a9140a9..31d75e2 100644 --- a/iquail/solution/__init__.py +++ b/iquail/solution/__init__.py @@ -3,3 +3,4 @@ from .solution_zip import SolutionZip from .solution_packed import SolutionPacked from .solution_github import SolutionGitHub +from .solution_fileserver import SolutionFileServer diff --git a/iquail/solution/solution_fileserver.py b/iquail/solution/solution_fileserver.py new file mode 100644 index 0000000..de4847c --- /dev/null +++ b/iquail/solution/solution_fileserver.py @@ -0,0 +1,161 @@ +import os +import sys +import shutil +from .solution_base import SolutionBase +from .solution_fileserver_wrapper import QuailFS +from ..builder.cmd_fileserver_client import CmdFileserverClient +from ..errors import SolutionUnreachableError +from ..errors import SolutionFileNotFoundError +from ..errors import SolutionVersionNotFoundError +from ..errors import SolutionDecompressionError +from ..helper import misc +from ..helper import OS_WINDOWS + +class SolutionFileServer(SolutionBase): + def __init__(self, host, port, build_path, fileserver_path=None): + super().__init__() + self._binary_name = 'iQuailClient' + if OS_WINDOWS: + self._binary_name += '.exe' + self._host = host + self._port = port + self._fileserver_path = fileserver_path + if fileserver_path: + self._fileserver_path = os.path.abspath(self._fileserver_path) + self._build_path = build_path + self._build_path = os.path.abspath(self._build_path) + if misc.running_from_script(): + self._client_bin_path = os.path.join(self._build_path, self._binary_name) + else: + self._client_bin_path = os.path.join(sys._MEIPASS, self._binary_name) + self._tmpdir = None + self._serv = None + self._files = None + self._nbrFiles = 1 + self._nbrFilesDownloaded = 0 + self._update = False + + def local(self): + return False + + def get_version_string(self): + if self._serv == None: + self._init_server() + version = self._serv.get_version() + self._fini_server() + return version + return self._serv.get_version() + + def _get_patch_version_name(self, curr, last): + return curr + '_TO_' + last + + def _init_server(self): + self._tmpdir = misc.safe_mkdtemp() + self._serv = QuailFS(self._client_bin_path, self._tmpdir) + if not self._serv.connect(self._host, self._port): + raise SolutionUnreachableError("FileServer.connect() failed: %s" % + self._serv.get_error()) + + def _fini_server(self): + self._serv.disconnect() + shutil.rmtree(self._tmpdir) + self._serv = None + + def open(self): + self._init_server() + currentVersion = self.get_installed_version() + lastVersion = self.get_version_string() + if currentVersion != None and currentVersion != lastVersion: + self._update = True + patchName = self._get_patch_version_name(currentVersion, lastVersion) + if not self._serv.set_version(patchName): + raise SolutionVersionNotFoundError("FileServer.set_version() failed: %s" % + self._serv.get_error()) + self._nbrFiles = self._serv.get_nbr_files() + self._nbrFilesDownloaded = 0 + + def close(self): + self._fini_server() + self._update = False + + def _parse_ls(self, lines): + dirs, files = [], [] + for line in lines: + ftype, fsize, fname = line.split(' ', 2) + if ftype == 'd': + dirs.append(fname) + else: + files.append(fname) + return (dirs, files) + + def _walk_rec(self, root): + lines = self._serv.ls(root) + dirs, files = self._parse_ls(lines) + yield (root, dirs, files) + for d in dirs: + yield from self._walk_rec(os.path.join(root, d)) + + def walk(self): + return self._walk_rec('.') + + def _get_tmp_path(self, relpath): + return os.path.join(self._tmpdir, relpath) + + def _decompress(self, sourcename, targetname, diffname): + def bytes_from_file(filename, chunksize=8192): + with open(filename, "rb") as f: + while True: + chunk = f.read(chunksize) + if chunk: + for b in chunk: + yield b + else: + break + target = open(targetname, 'w+b') + source = open(sourcename, 'rb') + buffer = b'' + for byte in bytes_from_file(diffname): + if byte == ord(b'\n') and len(buffer): + (header, arg) = buffer.split(b':', maxsplit=1) + if header == b'INSERT': + (off, _, length) = arg.split() + source.seek(int(off)) + target.write(source.read(int(length))) + elif header == b'COPY' and len(arg) > 1: + target.write(bytes([arg[1]])) + elif header == b'COPY': + target.write(bytes([byte])) + buffer = b'' + else: + if buffer == b'\n': + buffer = b'' + buffer += bytes([byte]) + target.close() + source.close() + + def _try_decompress(self, relpath): + source_path = self.retrieve_current_file(relpath) + if source_path == None: + return + diff_path = self._get_tmp_path(relpath) + target_path = diff_path + diff_path = diff_path + '_diff' + os.rename(target_path, diff_path) + self._decompress(source_path, target_path, diff_path) + os.remove(diff_path) + + def retrieve_file(self, relpath): + if not self._serv.get_file(relpath.replace(os.path.sep, '/')): + raise SolutionFileNotFoundError('FileServer.get_file() failed') + try: + self._try_decompress(relpath) + except Exception as e: + raise SolutionDecompressionError('Unexcpected error in decompression: ' + str(e)) + self._nbrFilesDownloaded += 1 + self._update_progress(percent=(100*self._nbrFilesDownloaded)/self._nbrFiles, status='downloading', log=relpath+'\n') + return self._get_tmp_path(relpath) + + def builder_cmds(self): + cmds = super().builder_cmds() + [CmdFileserverClient( + self._build_path, self._fileserver_path, self._binary_name)] + return cmds diff --git a/iquail/solution/solution_fileserver_wrapper.py b/iquail/solution/solution_fileserver_wrapper.py new file mode 100644 index 0000000..82458c5 --- /dev/null +++ b/iquail/solution/solution_fileserver_wrapper.py @@ -0,0 +1,75 @@ +import pexpect +import pexpect.popen_spawn + +class QuailFS: + def __init__(self, client_bin_path, dl_path): + self.pipe = None + self.error = '' + self.client_bin_path = client_bin_path + self.dl_path = dl_path + + def _send_cmd(self, cmd): + self.pipe.sendline(cmd) + self.pipe.expect('> ') + blist = list(self.pipe.before.splitlines()) + return [item.decode('utf-8') for item in blist] + + def _parse_error(self, lines, start='ERROR:'): + if len(lines) > 0 and lines[0].startswith(start): + self.error = lines[0] + return True + return False + + def connect(self, ip, port): + args = ' '.join([self.client_bin_path, ip, port, self.dl_path]) + try: + self.pipe = pexpect.popen_spawn.PopenSpawn(args) + except pexpect.exceptions.ExceptionPexpect: + self.error = 'Cannot open pipe' + return False + ret = self.pipe.expect(['> ', 'Couldn\'t connect to host', 'Exception:']) + if ret != 0: + if ret == 1: + self.error = 'Cannot connect' + else: + self.error = 'Cannot open download directory' + return False + return True + + def ls(self, path='.'): + lines = self._send_cmd('LS ' + path) + if self._parse_error(lines, 'LS failed'): + return None + return lines[1:] + + def get_version(self): + lines = self._send_cmd('VERSION GET') + return lines[1] + + def list_versions(self): + lines = self._send_cmd('VERSION LIST') + return lines[1:] + + def set_version(self, version): + lines = self._send_cmd('VERSION SET ' + version) + if self._parse_error(lines, 'Invalid command'): + return False + return True + + def get_file(self, path): + lines = self._send_cmd('GET_FILE ' + path) + if self._parse_error(lines, 'File not received'): + return False + return True + + def get_nbr_files(self): + lines = self._send_cmd('NBR_FILES') + return int(lines[0]) + + def get_error(self): + return self.error + + def disconnect(self): + self.pipe.sendline('EXIT') + self.pipe.expect(pexpect.EOF) + self.pipe = None