1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
| import glob
import tqdm
from multiprocessing import Process, Queue
import time
import math
import alog, os
from skimage import io
import argparse
m = Manager()
d = m.dict()
def chunks(l, n):
return [l[i:i+n] for i in range(0, len(l), n)]
def do_job(job_id, data_slice):
for filename in (data_slice):
try:
if not os.path.exists(filename):
print("not exist")
img = io.imread(filename)
except:
print(filename)
def dispatch_jobs(data, job_number):
total = len(data)
chunk_size = math.ceil(total / job_number)
slice = chunks(data, chunk_size)
jobs = []
for i, s in enumerate(slice):
j = Process(target=do_job, args=(i, s))
j.daemon = True
jobs.append(j)
for j in jobs:
j.start()
exit_codes = [p.join() for p in jobs]
print("finish")
return exit_codes
if __name__ == "__main__":
parser = argparse.ArgumentParser(description='Process some integers.')
parser.add_argument('--path', type=str,
help='path')
args = parser.parse_args()
print("waiting...")
img_path = os.path.join(args.path, "*.jpg")
data = [x for x in glob.glob(img_path)]
dispatch_jobs(data, 40)
|