diff --git a/.ci/release.sh b/.ci/release.sh index 55618bd727f86e43a7b9cfe7a260dac1ae0266a6..c6e1620a20340a5d686781031cbbe4bb8a81f1e2 100755 --- a/.ci/release.sh +++ b/.ci/release.sh @@ -2,6 +2,7 @@ set -e +git clean -fxd . _version=$(git describe --exact-match --tags HEAD || echo $(poetry version -s).dev$(date +%Y%m%d)) VERSION=${_version#'v'} # remove prefixed 'v' if any cp pyproject.toml /tmp/pyproject.toml.1 diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 010e7722f99a9e8db3d0e2c57fab0ce4c2499c4a..e8dfc3595408c488a0b244cf05bf042a69180fb5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -5,7 +5,7 @@ stages: - deploy before_script: - - apt update && apt install -y make + - apt update && apt install -y make git build: stage: build diff --git a/CHANGELOG.md b/CHANGELOG.md index 7507d96962cdd2f61ec060f17b62e10ebfc01389..c124128e9dd54e0c615f84b9eeea11a55fb2850f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Support for `create`, `logs` and `submit` endpoint. - Support for `BITIA_SERVER` environment variable. - Setting log level from the command line -- Session support. Same connection is used multiple time during the session. +- Session support. Same connection can be used throughout the session. ## [0.1.3] - 2022-09-29 ### Added diff --git a/bitia/dsl.py b/bitia/dsl.py new file mode 100644 index 0000000000000000000000000000000000000000..56bb22d4fd383fa26ea3b8ac8a3ae3b8b3617bf7 --- /dev/null +++ b/bitia/dsl.py @@ -0,0 +1,107 @@ +__author__ = "Dilawar Singh" +__email__ = "dilawar@subcom.tech" + +import typing as T +from urllib.parse import urlparse + +import pyparsing as pp + +ppc = pp.pyparsing_common + + +class FileWithLocation: + def __init__(self, filename: str, location: T.Optional[str] = None): + self.filename = filename + self.location = location + + def __repr__(self): + return str((self.filename, self.location)) + + def url(self): + return self.location.geturl() + + @staticmethod + def from_tokens(_s, _loc, toks): + loc = toks[-1] + assert 1 <= len(toks) <= 2, f"Either 1 or 2 tokens, got {toks}" + filename = loc.path if len(toks) == 1 else str(toks[0]) + return FileWithLocation(filename, loc) + + +p_start = pp.Literal("<").suppress() +p_end = pp.Literal(">").suppress() +p_pipe = pp.Literal("|").suppress() + +# fixme: filename can have no-printables? +p_filepath = pp.Word(pp.printables.translate({ord(i): None for i in r"|"})).set_name( + "filepath" +) + +p_url = pp.Word(pp.printables.translate({ord(i): None for i in r"<>"})).set_name("url") +p_url.setParseAction(lambda xs: list(map(urlparse, xs))) + +p_file_with_url = p_start + pp.Optional(p_filepath + p_pipe) + p_url + p_end +p_file_with_url.set_name("file_with_location") +p_file_with_url.set_parse_action(FileWithLocation.from_tokens) +# p_file_with_url.set_debug() + + +class DSL: + """Domain Spefic Langauge for BiTIA""" + + def __init__(self): + pass + + @staticmethod + def transform_line(line: str) -> str: + """Transform a simple line""" + # rule 1: search for p_file_with_url and replace those. + to_replace: T.Dict[str, str] = {} + download_lines = set() + for tokens, start, end in p_file_with_url.scan_string(line): + download_lines.add(f"bitia-runner tools make-available {tokens[0].url()}") + to_replace[line[start:end]] = tokens[0].filename + for k, v in to_replace.items(): + line = line.replace(k, v) + return "\n".join(download_lines) + "\n" + line + + +def test_simple_cli_with_dsl(): + """Test the full dsl""" + lines = [ + "# samtools sort -n Col0_C1.100k.bam -o out.sort.bam", + "samtools sort -n <Col0_C1.100k.bam|https://figshare.com/ndownloader/files/2133244> -o out.sort.bam", + "# samtools fastq Col0_C1.100k.bam", + "samtools fastq <Col0_C1.100k.bam|https://figshare.com/ndownloader/files/2133244>", + "# fastqc SRR20076358_1.fastq.gz", + "fastqc <ftp://ftp.sra.ebi.ac.uk/vol1/fastq/SRR200/058/SRR20076358/SRR20076358_1.fastq.gz>", + ] + for line in lines: + newline = DSL.transform_line(line) + print("\n--") + print(f"\told={line}\n\tnew={newline}") + + +def test_parser(): + """tests""" + assert not p_start.parse_string("<") + assert not p_end.parse_string(">") + a = p_url.parse_string("https://subcom.tech/a.txt#sha256=3319eqeqda") + print(11, a) + assert a + assert a[0].scheme == "https" + assert a[0].fragment == "sha256=3319eqeqda" + assert a[0].path == "/a.txt" + + a = p_file_with_url.parse_string("<foo.txt|http://example.com/foo.txt>") + print(12, a) + assert a + + a = p_file_with_url.parse_string("<http://example.com/foo.txt>") + assert a + print(a) + + +if __name__ == "__main__": + test_simple_cli_with_dsl() + test_parser() diff --git a/bitia/logger.py b/bitia/logger.py index 70b44abb3fdf9ca4b3dfbeb2fdebf9299848fed4..98e5e4f71c0227ab6dcc17bf8697c2cd52ea6553 100644 --- a/bitia/logger.py +++ b/bitia/logger.py @@ -1,4 +1,5 @@ import logging + from rich.logging import RichHandler from rich.console import Console diff --git a/bitia/pipeline.py b/bitia/pipeline.py index 4e9e532b265bb0b6cc33441ef7d3ad054f1b67f3..867c7031a57225c559cb9146d662554610b35a6f 100644 --- a/bitia/pipeline.py +++ b/bitia/pipeline.py @@ -11,6 +11,7 @@ from pathlib import Path # from datetime import datetime import validators +from bitia import version import bitia.config as bconfig from bitia.common import sha256sum, dir_info from bitia.logger import logger @@ -71,6 +72,8 @@ class Pipeline: pipeline_file = pipeline_dir / bconfig.BITIA_MAIN_SCRIPT_NAME lines: T.List[str] = [] lines.append("#!/bin/sh") + lines.append(f"# generated by bitia-cli version: {version()}") + lines.append("# (c) 2022-, Subconscious Compute") assert "\n" not in comment, "An eol char is found in comment" lines.append(f"# {comment}") lines.append(f"{cmd}") diff --git a/pyproject.toml b/pyproject.toml index d7320bf1bc6ab5b1846173a7983c1a5a6e617f4c..d18e459cbeee1953fbed02df7eb9dfe880ec63f2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "bitia" -version = "0.2.0b1" +version = "0.2.0" description = "BioInformatics Tool for Infrastructure Automation (BiTIA) CLI utility." authors = ["Dilawar Singh <dilawar@subcom.tech>"] readme = "README.md" @@ -11,6 +11,7 @@ requests = "^2.28.1" typer = "^0.6.1" rich = "^12.5.1" validators = "^0.20.0" +pyparsing = "^3.0.9" [tool.poetry.scripts] bitia = "bitia.__main__:app"