Skip to content

Tool functions

Common tool functions

  • read_json(file_path): Read json data into python dict
  • write_json(content, file_path): Write dict into json file
  • read_pickle(file_path): Read content of pickle file
  • write_pickle(content, file_path): Write content to pickle file
  • execute_cmd(cmd): Execute given shell command
  • gen_md5(string_or_file_path): Generate md5 value of a given file path or a string
  • download_file(url, save_path): Download file for a given link

download_file(url, save_path='download_file', **kwargs)

Download file for a given link

Parameters:

Name Type Description Default
url str

download link

 required
save_path Union[str, Path]

file save path

 'download_file'
**kwargs

session(option): if provide, download link from session

 {}

Returns:

Type Description
bool

download result: True if download success else False
Source code in cptools\function.py 
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
def download_file(url: str, save_path: Union[str, Path] = "download_file", **kwargs) -> bool:
    """Download file for a given link

    Args:
        url: download link
        save_path: file save path
        **kwargs: `session`(option): if provide, download link from session

    Returns:
        download result: True if download success else False
    """
    session = kwargs.get('session', requests)
    if session:
        res = session.get(url, stream=True)
    else:
        res = requests.get(url, stream=True)
    file_size = int(res.headers['content-length'])
    chunk_size = 1024
    if res.status_code == 200:
        progress_bar = tqdm(
            total=file_size, initial=0, unit='B', unit_scale=True,
        )
        with open(save_path, 'wb') as f:
            for data in res.iter_content(chunk_size=chunk_size):
                if data:
                    f.write(data)
                    progress_bar.update(chunk_size)
        progress_bar.close()
        return True
    else:
        return False

execute_cmd(cmd, timeout=900)

Execute given shell command

Parameters:

Name Type Description Default
cmd Union[str, list]

command, could be a list or string. e.g., ['ls', '-l'] or 'ls -l'

 required
timeout int 900

Returns:

Type Description
dict

A dict of execute result, including errcode and errmsg(if errcode==0, errmsg is output)
Source code in cptools\function.py 
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
def execute_cmd(cmd: Union[str, list], timeout: int = 900) -> dict:
    """Execute given shell command
    Args:
        cmd: command, could be a list or string. e.g., ['ls', '-l'] or 'ls -l'
        timeout:

    Returns:
        A dict of execute result, including errcode and errmsg(if errcode==0, errmsg is output)
    """
    try:
        p = subprocess.run(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
                           timeout=timeout)

    except subprocess.TimeoutExpired as e:
        return {
            'errcode': 401,
            'errmsg': 'timeout'
        }
    return {
        'errcode': p.returncode,
        'errmsg': p.stdout.decode()
    }

gen_md5(string_or_file_path)

Generate md5 value of a given file path or a string

Parameters:

Name Type Description Default
string_or_file_path Union[str, Path]

string literal or a file path

 required

Returns:

Type Description
str

md5
Source code in cptools\function.py 
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
def gen_md5(string_or_file_path: Union[str, Path]) -> str:
    """Generate md5 value of a given file path or a string

    Args:
        string_or_file_path: string literal or a file path

    Returns:
        md5

    """
    if Path(string_or_file_path).is_file():
        with open(string_or_file_path, 'rb') as f:
            m = hashlib.md5()
            chunk = f.read(4096)
            while chunk:
                m.update(chunk)
                chunk = f.read(4096)
    else:
        m = hashlib.md5()
        m.update(string_or_file_path.encode())
    return m.hexdigest()

read_json(file_path, by_line=False, **kwargs)

Read json data into python dict

Parameters:

Name Type Description Default
file_path Union[str, Path]

file save path

 required
by_line bool

if True, read data line by line

 False
**kwargs

other parameters used in open()

 {}

Returns:

Type Description
OrderedDict

json content
Source code in cptools\function.py 
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
def read_json(file_path: Union[str, Path], by_line: bool = False, **kwargs) -> OrderedDict:
    """Read json data into python dict
    Args:
        file_path: file save path
        by_line: if True, read data line by line
        **kwargs: other parameters used in open()
    Returns:
        json content
    """
    file_path = Path(file_path)
    with file_path.open('rt', **kwargs) as handle:
        if by_line:
            for line in handle:
                yield json.loads(line)
        else:
            return json.load(handle, object_hook=OrderedDict)

read_pickle(file_path, **kwargs)

Read content of pickle file

Parameters:

Name Type Description Default
file_path Union[str, Path]

file save path

 required
**kwargs

other parameters used in open()

 {}

Returns:

Type Description
object

content of pickle file
Source code in cptools\function.py 
64
65
66
67
68
69
70
71
72
73
74
75
def read_pickle(file_path: Union[str, Path], **kwargs) -> object:
    """Read content of pickle file
    Args:
        file_path: file save path
        **kwargs: other parameters used in open()
    Returns:
        content of pickle file

    """
    file_path = Path(file_path)
    with file_path.open('rb', **kwargs) as handle:
        return pkl.load(handle)

write_json(content, file_path, by_line=False, **kwargs)

Write dict into json file

Parameters:

Name Type Description Default
content dict

data dict

 required
file_path Union[str, Path]

file save path

 required
by_line bool

if True, write data line by line

 False
**kwargs

other parameters used in open()

 {}

Returns:

Type Description

None
Source code in cptools\function.py 
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
def write_json(content: dict, file_path: Union[str, Path], by_line: bool = False, **kwargs):
    """Write dict into json file
    Args:
        content: data dict
        file_path: file save path
        by_line: if True, write data line by line
        **kwargs: other parameters used in open()
    Returns:
        None
    """
    file_path = Path(file_path)
    with file_path.open('wt', **kwargs) as handle:
        if by_line:
            for line_data in content:
                handle.write(line_data + '\n')
        else:
            json.dump(content, handle, indent=4, sort_keys=True)

write_pickle(content, file_path, **kwargs)

Write content to pickle file

Parameters:

Name Type Description Default
content object

python object

 required
file_path Union[str, Path]

file save path

 required
**kwargs

other parameters used in open()

 {}

Returns:

Type Description

None
Source code in cptools\function.py 
78
79
80
81
82
83
84
85
86
87
88
89
def write_pickle(content: object, file_path: Union[str, Path], **kwargs):
    """Write content to pickle file
    Args:
        content: python object
        file_path: file save path
        **kwargs: other parameters used in open()
    Returns:
        None
    """
    file_path = Path(file_path)
    with file_path.open('wb', **kwargs) as handle:
        pkl.dump(content, handle)