Compare commits
4 Commits
b0fad56b13
...
7c6710469b
Author | SHA1 | Date |
---|---|---|
Jason Zhu | 7c6710469b | |
Jason Zhu | c4dde2cc3f | |
Jason Zhu | 2b8fdd8660 | |
Jason Zhu | 43a07fdf59 |
|
@ -0,0 +1,22 @@
|
||||||
|
# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.187.0/containers/python-3/.devcontainer/base.Dockerfile
|
||||||
|
|
||||||
|
# [Choice] Python version: 3, 3.9, 3.8, 3.7, 3.6
|
||||||
|
ARG VARIANT="3.9"
|
||||||
|
FROM mcr.microsoft.com/vscode/devcontainers/python:0-${VARIANT}
|
||||||
|
|
||||||
|
# [Option] Install Node.js
|
||||||
|
ARG INSTALL_NODE="true"
|
||||||
|
ARG NODE_VERSION="lts/*"
|
||||||
|
RUN if [ "${INSTALL_NODE}" = "true" ]; then su vscode -c "umask 0002 && . /usr/local/share/nvm/nvm.sh && nvm install ${NODE_VERSION} 2>&1"; fi
|
||||||
|
|
||||||
|
# [Optional] If your pip requirements rarely change, uncomment this section to add them to the image.
|
||||||
|
# COPY requirements.txt /tmp/pip-tmp/
|
||||||
|
# RUN pip3 --disable-pip-version-check --no-cache-dir install -r /tmp/pip-tmp/requirements.txt \
|
||||||
|
# && rm -rf /tmp/pip-tmp
|
||||||
|
|
||||||
|
# [Optional] Uncomment this section to install additional OS packages.
|
||||||
|
RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
|
||||||
|
&& apt-get -y install --no-install-recommends tree
|
||||||
|
|
||||||
|
# [Optional] Uncomment this line to install global node packages.
|
||||||
|
# RUN su vscode -c "source /usr/local/share/nvm/nvm.sh && npm install -g <your-package-here>" 2>&1
|
|
@ -0,0 +1,64 @@
|
||||||
|
// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at:
|
||||||
|
// https://github.com/microsoft/vscode-dev-containers/tree/v0.187.0/containers/python-3
|
||||||
|
{
|
||||||
|
"name": "Python 3",
|
||||||
|
"build": {
|
||||||
|
"dockerfile": "Dockerfile",
|
||||||
|
"context": "..",
|
||||||
|
"args": {
|
||||||
|
// Update 'VARIANT' to pick a Python version: 3, 3.6, 3.7, 3.8, 3.9
|
||||||
|
"VARIANT": "3.8",
|
||||||
|
// Options
|
||||||
|
"INSTALL_NODE": "false",
|
||||||
|
"NODE_VERSION": "lts/*"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
|
||||||
|
// Set *default* container specific settings.json values on container create.
|
||||||
|
"settings": {
|
||||||
|
"python.pythonPath": "/usr/local/bin/python",
|
||||||
|
"python.languageServer": "Pylance",
|
||||||
|
"python.linting.enabled": true,
|
||||||
|
"python.linting.pylintEnabled": true,
|
||||||
|
"python.formatting.autopep8Path": "/usr/local/py-utils/bin/autopep8",
|
||||||
|
"python.formatting.blackPath": "/usr/local/py-utils/bin/black",
|
||||||
|
"python.formatting.yapfPath": "/usr/local/py-utils/bin/yapf",
|
||||||
|
"python.linting.banditPath": "/usr/local/py-utils/bin/bandit",
|
||||||
|
"python.linting.flake8Path": "/usr/local/py-utils/bin/flake8",
|
||||||
|
"python.linting.mypyPath": "/usr/local/py-utils/bin/mypy",
|
||||||
|
"python.linting.pycodestylePath": "/usr/local/py-utils/bin/pycodestyle",
|
||||||
|
"python.linting.pydocstylePath": "/usr/local/py-utils/bin/pydocstyle",
|
||||||
|
"python.linting.pylintPath": "/usr/local/py-utils/bin/pylint",
|
||||||
|
|
||||||
|
// customerized
|
||||||
|
"terminal.integrated.automationShell.linux": "/bin/zsh",
|
||||||
|
},
|
||||||
|
|
||||||
|
// Add the IDs of extensions you want installed when the container is created.
|
||||||
|
"extensions": [
|
||||||
|
"ms-python.python",
|
||||||
|
"ms-python.vscode-pylance",
|
||||||
|
"visualstudioexptteam.vscodeintellicode",
|
||||||
|
|
||||||
|
// git
|
||||||
|
"piotrpalarz.vscode-gitignore-generator",
|
||||||
|
"mhutchie.git-graph",
|
||||||
|
"eamodio.gitlens",
|
||||||
|
|
||||||
|
// python
|
||||||
|
"njpwerner.autodocstring",
|
||||||
|
|
||||||
|
// editor
|
||||||
|
"alefragnani.numbered-bookmarks",
|
||||||
|
|
||||||
|
],
|
||||||
|
|
||||||
|
// Use 'forwardPorts' to make a list of ports inside the container available locally.
|
||||||
|
// "forwardPorts": [],
|
||||||
|
|
||||||
|
// Use 'postCreateCommand' to run commands after the container is created.
|
||||||
|
"postCreateCommand": "pip3 install --user -r requirements.txt",
|
||||||
|
|
||||||
|
// Comment out connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.
|
||||||
|
"remoteUser": "vscode"
|
||||||
|
}
|
|
@ -1,4 +1,52 @@
|
||||||
# ---> Python
|
# File created using '.gitignore Generator' for Visual Studio Code: https://bit.ly/vscode-gig
|
||||||
|
|
||||||
|
# Created by https://www.toptal.com/developers/gitignore/api/visualstudiocode,linux,macos,python,windows
|
||||||
|
# Edit at https://www.toptal.com/developers/gitignore?templates=visualstudiocode,linux,macos,python,windows
|
||||||
|
|
||||||
|
### Linux ###
|
||||||
|
*~
|
||||||
|
|
||||||
|
# temporary files which can be created if a process still has a handle open of a deleted file
|
||||||
|
.fuse_hidden*
|
||||||
|
|
||||||
|
# KDE directory preferences
|
||||||
|
.directory
|
||||||
|
|
||||||
|
# Linux trash folder which might appear on any partition or disk
|
||||||
|
.Trash-*
|
||||||
|
|
||||||
|
# .nfs files are created when an open file is removed but is still being accessed
|
||||||
|
.nfs*
|
||||||
|
|
||||||
|
### macOS ###
|
||||||
|
# General
|
||||||
|
.DS_Store
|
||||||
|
.AppleDouble
|
||||||
|
.LSOverride
|
||||||
|
|
||||||
|
# Icon must end with two \r
|
||||||
|
Icon
|
||||||
|
|
||||||
|
# Thumbnails
|
||||||
|
._*
|
||||||
|
|
||||||
|
# Files that might appear in the root of a volume
|
||||||
|
.DocumentRevisions-V100
|
||||||
|
.fseventsd
|
||||||
|
.Spotlight-V100
|
||||||
|
.TemporaryItems
|
||||||
|
.Trashes
|
||||||
|
.VolumeIcon.icns
|
||||||
|
.com.apple.timemachine.donotpresent
|
||||||
|
|
||||||
|
# Directories potentially created on remote AFP share
|
||||||
|
.AppleDB
|
||||||
|
.AppleDesktop
|
||||||
|
Network Trash Folder
|
||||||
|
Temporary Items
|
||||||
|
.apdisk
|
||||||
|
|
||||||
|
### Python ###
|
||||||
# Byte-compiled / optimized / DLL files
|
# Byte-compiled / optimized / DLL files
|
||||||
__pycache__/
|
__pycache__/
|
||||||
*.py[cod]
|
*.py[cod]
|
||||||
|
@ -138,3 +186,49 @@ dmypy.json
|
||||||
# Cython debug symbols
|
# Cython debug symbols
|
||||||
cython_debug/
|
cython_debug/
|
||||||
|
|
||||||
|
### VisualStudioCode ###
|
||||||
|
.vscode/*
|
||||||
|
!.vscode/settings.json
|
||||||
|
!.vscode/tasks.json
|
||||||
|
!.vscode/launch.json
|
||||||
|
!.vscode/extensions.json
|
||||||
|
*.code-workspace
|
||||||
|
|
||||||
|
# Local History for Visual Studio Code
|
||||||
|
.history/
|
||||||
|
|
||||||
|
### VisualStudioCode Patch ###
|
||||||
|
# Ignore all local history of files
|
||||||
|
.history
|
||||||
|
.ionide
|
||||||
|
|
||||||
|
### Windows ###
|
||||||
|
# Windows thumbnail cache files
|
||||||
|
Thumbs.db
|
||||||
|
Thumbs.db:encryptable
|
||||||
|
ehthumbs.db
|
||||||
|
ehthumbs_vista.db
|
||||||
|
|
||||||
|
# Dump file
|
||||||
|
*.stackdump
|
||||||
|
|
||||||
|
# Folder config file
|
||||||
|
[Dd]esktop.ini
|
||||||
|
|
||||||
|
# Recycle Bin used on file shares
|
||||||
|
$RECYCLE.BIN/
|
||||||
|
|
||||||
|
# Windows Installer files
|
||||||
|
*.cab
|
||||||
|
*.msi
|
||||||
|
*.msix
|
||||||
|
*.msm
|
||||||
|
*.msp
|
||||||
|
|
||||||
|
# Windows shortcuts
|
||||||
|
*.lnk
|
||||||
|
|
||||||
|
# End of https://www.toptal.com/developers/gitignore/api/visualstudiocode,linux,macos,python,windows
|
||||||
|
|
||||||
|
# Custom rules (everything added below won't be overriden by 'Generate .gitignore File' if you use 'Update' option)
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,18 @@
|
||||||
|
{
|
||||||
|
"python.linting.enabled": true,
|
||||||
|
"python.linting.pylintEnabled": false,
|
||||||
|
"python.linting.mypyEnabled": true,
|
||||||
|
"python.testing.unittestArgs": [
|
||||||
|
"-v",
|
||||||
|
"-s",
|
||||||
|
".",
|
||||||
|
"-p",
|
||||||
|
"test*.py"
|
||||||
|
],
|
||||||
|
"python.testing.pytestEnabled": true,
|
||||||
|
"python.testing.nosetestsEnabled": false,
|
||||||
|
"python.testing.unittestEnabled": false,
|
||||||
|
"python.testing.pytestArgs": [
|
||||||
|
"tests"
|
||||||
|
]
|
||||||
|
}
|
|
@ -1,3 +1,7 @@
|
||||||
# synology_link_manipultor
|
# synology_link_manipultor
|
||||||
|
|
||||||
Used for converting softlink and hardlink
|
Used for converting softlink and hardlink
|
||||||
|
|
||||||
|
Development Plan:
|
||||||
|
* [x] Setup Environment: pytest + mypy + vscode extensions
|
||||||
|
* [ ] Create unit test and integration tests (pytest) for hdlink_to_symlinks
|
|
@ -0,0 +1,10 @@
|
||||||
|
import os
|
||||||
|
import pandas as pd
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
|
||||||
|
def append_col_inode(df: pd.DataFrame) -> pd.DataFrame:
|
||||||
|
col_inode = df.apply(lambda x: os.stat(x).st_nlink,
|
||||||
|
axis=1,
|
||||||
|
columns=['path'])
|
||||||
|
return df.assign(inode = col_inode)
|
|
@ -0,0 +1,37 @@
|
||||||
|
import pathlib
|
||||||
|
import typing
|
||||||
|
import os
|
||||||
|
|
||||||
|
class Media:
|
||||||
|
|
||||||
|
def __init__(self, path: typing.Union[str, pathlib.Path]):
|
||||||
|
self.path = pathlib.Path(path)
|
||||||
|
|
||||||
|
def exists(self) -> bool:
|
||||||
|
return self.path.exists()
|
||||||
|
|
||||||
|
def is_symlink(self) -> bool:
|
||||||
|
return self.path.is_symlink()
|
||||||
|
|
||||||
|
def number_of_links(self) -> int:
|
||||||
|
return self.path.lstat().st_nlink
|
||||||
|
|
||||||
|
def inode(self) -> int:
|
||||||
|
return self.path.lstat().st_ino
|
||||||
|
|
||||||
|
def relative_to_parents(self, target: pathlib.Path) -> str:
|
||||||
|
path_parent = self.path.resolve().parent
|
||||||
|
target_parent = target.resolve().parent
|
||||||
|
|
||||||
|
return os.path.relpath(path=target_parent, start=path_parent)
|
||||||
|
|
||||||
|
def symlink_to(self, target: pathlib.Path):
|
||||||
|
self.path.symlink_to(target=os.path.join(self.relative_to_parents(target=target), target.name),
|
||||||
|
target_is_directory=False)
|
||||||
|
|
||||||
|
def convert_symlink_to_hdlink(self):
|
||||||
|
if self.is_symlink() is True:
|
||||||
|
src = self.path.resolve()
|
||||||
|
dst = self.path
|
||||||
|
os.remove(dst)
|
||||||
|
os.link(src=src, dst=dst)
|
|
@ -0,0 +1,13 @@
|
||||||
|
attrs==21.2.0
|
||||||
|
iniconfig==1.1.1
|
||||||
|
numpy==1.21.0
|
||||||
|
packaging==21.0
|
||||||
|
pandas==1.3.0
|
||||||
|
pluggy==0.13.1
|
||||||
|
py==1.10.0
|
||||||
|
pyparsing==2.4.7
|
||||||
|
pytest==6.2.4
|
||||||
|
python-dateutil==2.8.2
|
||||||
|
pytz==2021.1
|
||||||
|
six==1.16.0
|
||||||
|
toml==0.10.2
|
|
@ -0,0 +1,26 @@
|
||||||
|
import pathlib
|
||||||
|
import argparse
|
||||||
|
|
||||||
|
from media import Media
|
||||||
|
|
||||||
|
def find_all_files(directory: pathlib.Path):
|
||||||
|
ps = []
|
||||||
|
for p in directory.rglob('*'):
|
||||||
|
if "eaDir" not in str(p):
|
||||||
|
ps.append((p))
|
||||||
|
return ps
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
parser = argparse.ArgumentParser(description='convert hardlink to symlink in download folder')
|
||||||
|
|
||||||
|
parser.add_argument('Library',
|
||||||
|
metavar='--library',
|
||||||
|
type=str,
|
||||||
|
help='the path to library')
|
||||||
|
|
||||||
|
args = parser.parse_args()
|
||||||
|
path_library = args.Library
|
||||||
|
paths = find_all_files(pathlib.Path(path_library))
|
||||||
|
|
||||||
|
for x in paths:
|
||||||
|
Media(x).convert_symlink_to_hdlink()
|
|
@ -0,0 +1,2 @@
|
||||||
|
file/
|
||||||
|
target/
|
|
@ -0,0 +1,29 @@
|
||||||
|
import pytest
|
||||||
|
from pathlib import Path
|
||||||
|
import os
|
||||||
|
import pandas as pd
|
||||||
|
import shutil
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def cleanup():
|
||||||
|
dir_delete = [Path('./tests/file/'), Path('./tests/target/')]
|
||||||
|
|
||||||
|
for d in dir_delete:
|
||||||
|
shutil.rmtree(str(d), ignore_errors=True)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def create_test_files(cleanup) -> pd.DataFrame:
|
||||||
|
|
||||||
|
df = pd.DataFrame(columns=['path','target'])
|
||||||
|
Path('./tests/file/').mkdir(parents=True, exist_ok=False)
|
||||||
|
Path('./tests/target/').mkdir(parents=True, exist_ok=False)
|
||||||
|
|
||||||
|
for x in range(1,6):
|
||||||
|
p = Path('./tests/file/file' + str(x) + '.txt')
|
||||||
|
t = Path('./tests/target/target' + str(x) + '.txt')
|
||||||
|
t.touch(exist_ok=False)
|
||||||
|
p.symlink_to(t,target_is_directory=False)
|
||||||
|
df.append({'path': p, 'target': t} , ignore_index=True)
|
||||||
|
|
||||||
|
return df
|
|
@ -0,0 +1,67 @@
|
||||||
|
import pytest
|
||||||
|
from pathlib import Path
|
||||||
|
import shutil
|
||||||
|
import os, sys
|
||||||
|
|
||||||
|
sys.path.insert(1, os.path.abspath('.'))
|
||||||
|
|
||||||
|
from test_helper import create_test_files, cleanup
|
||||||
|
from media import Media
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def create_one_pair_links(cleanup):
|
||||||
|
Path('./tests/target/').mkdir(parents=True, exist_ok=False)
|
||||||
|
Path('./tests/file').mkdir(parents=True, exist_ok=True)
|
||||||
|
p = Path('./tests/file/file1.txt')
|
||||||
|
t = Path('./tests/target/target1.txt')
|
||||||
|
t.touch(exist_ok=False)
|
||||||
|
p.symlink_to(Path('../target/target1.txt'))
|
||||||
|
|
||||||
|
def test_media_symlink_exist(create_one_pair_links):
|
||||||
|
p = Media('./tests/file/file1.txt')
|
||||||
|
assert p.exists()
|
||||||
|
assert p.is_symlink()
|
||||||
|
|
||||||
|
def test_media_target_exist(create_one_pair_links):
|
||||||
|
t = Media('./tests/target/target1.txt')
|
||||||
|
assert t.exists()
|
||||||
|
assert not t.is_symlink()
|
||||||
|
assert t.number_of_links() == 1
|
||||||
|
|
||||||
|
def test_media_symlink(create_one_pair_links):
|
||||||
|
|
||||||
|
p = Media('./tests/file/file1.txt')
|
||||||
|
t = Media('./tests/target/target1.txt')
|
||||||
|
|
||||||
|
text_in = 'hello, world'
|
||||||
|
|
||||||
|
with open(p.path, 'w') as writer:
|
||||||
|
writer.write(text_in)
|
||||||
|
|
||||||
|
with open(t.path, 'r') as reader:
|
||||||
|
text_out = reader.read()
|
||||||
|
assert text_in == text_out
|
||||||
|
|
||||||
|
p2 = Media('./tests/file/file2.txt')
|
||||||
|
assert p2.relative_to_parents(target=t.path) == '../target'
|
||||||
|
p2.symlink_to(target=t.path)
|
||||||
|
with open(p2.path, 'r') as reader:
|
||||||
|
text_out = reader.read()
|
||||||
|
assert text_in == text_out
|
||||||
|
|
||||||
|
def test_media_symlink_to_hdlink(create_one_pair_links):
|
||||||
|
|
||||||
|
p = Media('./tests/file/file1.txt')
|
||||||
|
t = Media('./tests/target/target1.txt')
|
||||||
|
|
||||||
|
text_in = 'hello, world'
|
||||||
|
|
||||||
|
p.convert_symlink_to_hdlink()
|
||||||
|
assert not p.is_symlink()
|
||||||
|
|
||||||
|
with open(p.path, 'w') as writer:
|
||||||
|
writer.write(text_in)
|
||||||
|
|
||||||
|
with open(t.path, 'r') as reader:
|
||||||
|
text_out = reader.read()
|
||||||
|
assert text_out == text_in
|
|
@ -0,0 +1,23 @@
|
||||||
|
import pytest
|
||||||
|
import pathlib
|
||||||
|
|
||||||
|
from test_helper import cleanup
|
||||||
|
from media import Media
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def create_test_files(cleanup):
|
||||||
|
pathlib.Path('./tests/target/').mkdir(parents=True, exist_ok=False)
|
||||||
|
pathlib.Path('./tests/file').mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
for x in range(1,6):
|
||||||
|
p = Media('./tests/file/file' + str(x) + '.txt')
|
||||||
|
t = Media('./tests/target/target' + str(x) + '.txt')
|
||||||
|
t.path.touch(exist_ok=False)
|
||||||
|
p.symlink_to(t.path)
|
||||||
|
|
||||||
|
def test_symlink_to_hdlink(create_test_files):
|
||||||
|
for p in pathlib.Path('./tests/file/').iterdir():
|
||||||
|
Media(p).convert_symlink_to_hdlink()
|
||||||
|
|
||||||
|
assert True
|
||||||
|
# TODO: Modify test case to computationaly test inode of file
|
Loading…
Reference in New Issue