Skip to content
Snippets Groups Projects
Commit 966efc0c authored by dilawar's avatar dilawar :ant:
Browse files

Merge branch '8-implement-first-version-of-bitia-dsl-parser' into 'main'

Resolve "Implement first version of BiTIA DSL parser"

Closes #8

See merge request !12
parents 2baed570 89b228a7
No related branches found
No related tags found
1 merge request!12Resolve "Implement first version of BiTIA DSL parser"
Pipeline #4127 passed with stages
in 4 minutes and 19 seconds
...@@ -2,6 +2,7 @@ ...@@ -2,6 +2,7 @@
set -e set -e
git clean -fxd .
_version=$(git describe --exact-match --tags HEAD || echo $(poetry version -s).dev$(date +%Y%m%d)) _version=$(git describe --exact-match --tags HEAD || echo $(poetry version -s).dev$(date +%Y%m%d))
VERSION=${_version#'v'} # remove prefixed 'v' if any VERSION=${_version#'v'} # remove prefixed 'v' if any
cp pyproject.toml /tmp/pyproject.toml.1 cp pyproject.toml /tmp/pyproject.toml.1
......
...@@ -5,7 +5,7 @@ stages: ...@@ -5,7 +5,7 @@ stages:
- deploy - deploy
before_script: before_script:
- apt update && apt install -y make - apt update && apt install -y make git
build: build:
stage: build stage: build
......
...@@ -11,7 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ...@@ -11,7 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- Support for `create`, `logs` and `submit` endpoint. - Support for `create`, `logs` and `submit` endpoint.
- Support for `BITIA_SERVER` environment variable. - Support for `BITIA_SERVER` environment variable.
- Setting log level from the command line - Setting log level from the command line
- Session support. Same connection is used multiple time during the session. - Session support. Same connection can be used throughout the session.
## [0.1.3] - 2022-09-29 ## [0.1.3] - 2022-09-29
### Added ### Added
......
__author__ = "Dilawar Singh"
__email__ = "dilawar@subcom.tech"
import typing as T
from urllib.parse import urlparse
import pyparsing as pp
ppc = pp.pyparsing_common
class FileWithLocation:
def __init__(self, filename: str, location: T.Optional[str] = None):
self.filename = filename
self.location = location
def __repr__(self):
return str((self.filename, self.location))
def url(self):
return self.location.geturl()
@staticmethod
def from_tokens(_s, _loc, toks):
loc = toks[-1]
assert 1 <= len(toks) <= 2, f"Either 1 or 2 tokens, got {toks}"
filename = loc.path if len(toks) == 1 else str(toks[0])
return FileWithLocation(filename, loc)
p_start = pp.Literal("<").suppress()
p_end = pp.Literal(">").suppress()
p_pipe = pp.Literal("|").suppress()
# fixme: filename can have no-printables?
p_filepath = pp.Word(pp.printables.translate({ord(i): None for i in r"|"})).set_name(
"filepath"
)
p_url = pp.Word(pp.printables.translate({ord(i): None for i in r"<>"})).set_name("url")
p_url.setParseAction(lambda xs: list(map(urlparse, xs)))
p_file_with_url = p_start + pp.Optional(p_filepath + p_pipe) + p_url + p_end
p_file_with_url.set_name("file_with_location")
p_file_with_url.set_parse_action(FileWithLocation.from_tokens)
# p_file_with_url.set_debug()
class DSL:
"""Domain Spefic Langauge for BiTIA"""
def __init__(self):
pass
@staticmethod
def transform_line(line: str) -> str:
"""Transform a simple line"""
# rule 1: search for p_file_with_url and replace those.
to_replace: T.Dict[str, str] = {}
download_lines = set()
for tokens, start, end in p_file_with_url.scan_string(line):
download_lines.add(f"bitia-runner tools make-available {tokens[0].url()}")
to_replace[line[start:end]] = tokens[0].filename
for k, v in to_replace.items():
line = line.replace(k, v)
return "\n".join(download_lines) + "\n" + line
def test_simple_cli_with_dsl():
"""Test the full dsl"""
lines = [
"# samtools sort -n Col0_C1.100k.bam -o out.sort.bam",
"samtools sort -n <Col0_C1.100k.bam|https://figshare.com/ndownloader/files/2133244> -o out.sort.bam",
"# samtools fastq Col0_C1.100k.bam",
"samtools fastq <Col0_C1.100k.bam|https://figshare.com/ndownloader/files/2133244>",
"# fastqc SRR20076358_1.fastq.gz",
"fastqc <ftp://ftp.sra.ebi.ac.uk/vol1/fastq/SRR200/058/SRR20076358/SRR20076358_1.fastq.gz>",
]
for line in lines:
newline = DSL.transform_line(line)
print("\n--")
print(f"\told={line}\n\tnew={newline}")
def test_parser():
"""tests"""
assert not p_start.parse_string("<")
assert not p_end.parse_string(">")
a = p_url.parse_string("https://subcom.tech/a.txt#sha256=3319eqeqda")
print(11, a)
assert a
assert a[0].scheme == "https"
assert a[0].fragment == "sha256=3319eqeqda"
assert a[0].path == "/a.txt"
a = p_file_with_url.parse_string("<foo.txt|http://example.com/foo.txt>")
print(12, a)
assert a
a = p_file_with_url.parse_string("<http://example.com/foo.txt>")
assert a
print(a)
if __name__ == "__main__":
test_simple_cli_with_dsl()
test_parser()
import logging import logging
from rich.logging import RichHandler from rich.logging import RichHandler
from rich.console import Console from rich.console import Console
......
...@@ -11,6 +11,7 @@ from pathlib import Path ...@@ -11,6 +11,7 @@ from pathlib import Path
# from datetime import datetime # from datetime import datetime
import validators import validators
from bitia import version
import bitia.config as bconfig import bitia.config as bconfig
from bitia.common import sha256sum, dir_info from bitia.common import sha256sum, dir_info
from bitia.logger import logger from bitia.logger import logger
...@@ -71,6 +72,8 @@ class Pipeline: ...@@ -71,6 +72,8 @@ class Pipeline:
pipeline_file = pipeline_dir / bconfig.BITIA_MAIN_SCRIPT_NAME pipeline_file = pipeline_dir / bconfig.BITIA_MAIN_SCRIPT_NAME
lines: T.List[str] = [] lines: T.List[str] = []
lines.append("#!/bin/sh") lines.append("#!/bin/sh")
lines.append(f"# generated by bitia-cli version: {version()}")
lines.append("# (c) 2022-, Subconscious Compute")
assert "\n" not in comment, "An eol char is found in comment" assert "\n" not in comment, "An eol char is found in comment"
lines.append(f"# {comment}") lines.append(f"# {comment}")
lines.append(f"{cmd}") lines.append(f"{cmd}")
......
[tool.poetry] [tool.poetry]
name = "bitia" name = "bitia"
version = "0.2.0b1" version = "0.2.0"
description = "BioInformatics Tool for Infrastructure Automation (BiTIA) CLI utility." description = "BioInformatics Tool for Infrastructure Automation (BiTIA) CLI utility."
authors = ["Dilawar Singh <dilawar@subcom.tech>"] authors = ["Dilawar Singh <dilawar@subcom.tech>"]
readme = "README.md" readme = "README.md"
...@@ -11,6 +11,7 @@ requests = "^2.28.1" ...@@ -11,6 +11,7 @@ requests = "^2.28.1"
typer = "^0.6.1" typer = "^0.6.1"
rich = "^12.5.1" rich = "^12.5.1"
validators = "^0.20.0" validators = "^0.20.0"
pyparsing = "^3.0.9"
[tool.poetry.scripts] [tool.poetry.scripts]
bitia = "bitia.__main__:app" bitia = "bitia.__main__:app"
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment