Python源码示例：keras.utils.GeneratorEnqueuer()
示例1
def test_generator_enqueuer_threads():
enqueuer = GeneratorEnqueuer(create_generator_from_sequence_threads(
DummySequence([3, 200, 200, 3])), use_multiprocessing=False)
enqueuer.start(3, 10)
gen_output = enqueuer.get()
acc = []
for i in range(100):
acc.append(int(next(gen_output)[0, 0, 0, 0]))
"""
Not comparing the order since it is not guaranteed.
It may get ordered, but not a lot, one thread can take the GIL before he was supposed to.
"""
assert len(set(acc) - set(range(100))) == 0, "Output is not the same"
enqueuer.stop()
示例2
def test_generator_enqueuer_processes():
enqueuer = GeneratorEnqueuer(create_generator_from_sequence_pcs(
DummySequence([3, 200, 200, 3])), use_multiprocessing=True)
enqueuer.start(3, 10)
gen_output = enqueuer.get()
acc = []
for i in range(100):
acc.append(int(next(gen_output)[0, 0, 0, 0]))
assert acc != list(range(100)), "Order was keep in GeneratorEnqueuer with processes"
enqueuer.stop()
示例3
def test_generator_enqueuer_fail_threads():
enqueuer = GeneratorEnqueuer(create_generator_from_sequence_threads(
FaultSequence()), use_multiprocessing=False)
enqueuer.start(3, 10)
gen_output = enqueuer.get()
with pytest.raises(IndexError):
next(gen_output)
示例4
def test_generator_enqueuer_fail_processes():
enqueuer = GeneratorEnqueuer(create_generator_from_sequence_pcs(
FaultSequence()), use_multiprocessing=True)
enqueuer.start(3, 10)
gen_output = enqueuer.get()
with pytest.raises(IndexError):
next(gen_output)
示例5
def test_ordered_enqueuer_threads():
enqueuer = OrderedEnqueuer(DummySequence([3, 200, 200, 3]), use_multiprocessing=False)
enqueuer.start(3, 10)
gen_output = enqueuer.get()
acc = []
for i in range(100):
acc.append(next(gen_output)[0, 0, 0, 0])
assert acc == list(range(100)), "Order was not keep in GeneratorEnqueuer with threads"
enqueuer.stop()
示例6
def test_ordered_enqueuer_processes():
enqueuer = OrderedEnqueuer(DummySequence([3, 200, 200, 3]), use_multiprocessing=True)
enqueuer.start(3, 10)
gen_output = enqueuer.get()
acc = []
for i in range(100):
acc.append(next(gen_output)[0, 0, 0, 0])
assert acc == list(range(100)), "Order was not keep in GeneratorEnqueuer with processes"
enqueuer.stop()
示例7
def test_on_epoch_end_processes():
enqueuer = OrderedEnqueuer(DummySequence([3, 200, 200, 3]), use_multiprocessing=True)
enqueuer.start(3, 10)
gen_output = enqueuer.get()
acc = []
for i in range(200):
acc.append(next(gen_output)[0, 0, 0, 0])
assert acc[100:] == list([k * 5 for k in range(100)]), "Order was not keep in GeneratorEnqueuer with processes"
enqueuer.stop()
示例8
def test_on_epoch_end_threads():
enqueuer = OrderedEnqueuer(DummySequence([3, 200, 200, 3]), use_multiprocessing=False)
enqueuer.start(3, 10)
gen_output = enqueuer.get()
acc = []
for i in range(100):
acc.append(next(gen_output)[0, 0, 0, 0])
acc = []
for i in range(100):
acc.append(next(gen_output)[0, 0, 0, 0])
assert acc == list([k * 5 for k in range(100)]), "Order was not keep in GeneratorEnqueuer with processes"
enqueuer.stop()
示例9
def test_finite_generator_enqueuer_threads():
enqueuer = GeneratorEnqueuer(create_finite_generator_from_sequence_threads(
DummySequence([3, 200, 200, 3])), use_multiprocessing=False)
enqueuer.start(3, 10)
gen_output = enqueuer.get()
acc = []
for output in gen_output:
acc.append(int(output[0, 0, 0, 0]))
assert set(acc) == set(range(100)), "Output is not the same"
enqueuer.stop()
示例10
def test_finite_generator_enqueuer_processes():
enqueuer = GeneratorEnqueuer(create_finite_generator_from_sequence_pcs(
DummySequence([3, 200, 200, 3])), use_multiprocessing=True)
enqueuer.start(3, 10)
gen_output = enqueuer.get()
acc = []
for output in gen_output:
acc.append(int(output[0, 0, 0, 0]))
assert acc != list(range(100)), "Order was keep in GeneratorEnqueuer with processes"
enqueuer.stop()
示例11
def test_generator_enqueuer_processes():
enqueuer = GeneratorEnqueuer(create_generator_from_sequence_pcs(
DummySequence([3, 200, 200, 3])), use_multiprocessing=True)
enqueuer.start(3, 10)
gen_output = enqueuer.get()
acc = []
for i in range(100):
acc.append(int(next(gen_output)[0, 0, 0, 0]))
assert acc != list(range(100)), "Order was keep in GeneratorEnqueuer with processes"
enqueuer.stop()
示例12
def test_generator_enqueuer_fail_threads():
enqueuer = GeneratorEnqueuer(create_generator_from_sequence_threads(
FaultSequence()), use_multiprocessing=False)
enqueuer.start(3, 10)
gen_output = enqueuer.get()
with pytest.raises(IndexError):
next(gen_output)
示例13
def test_generator_enqueuer_fail_processes():
enqueuer = GeneratorEnqueuer(create_generator_from_sequence_pcs(
FaultSequence()), use_multiprocessing=True)
enqueuer.start(3, 10)
gen_output = enqueuer.get()
with pytest.raises(IndexError):
next(gen_output)
示例14
def test_ordered_enqueuer_threads():
enqueuer = OrderedEnqueuer(DummySequence([3, 200, 200, 3]), use_multiprocessing=False)
enqueuer.start(3, 10)
gen_output = enqueuer.get()
acc = []
for i in range(100):
acc.append(next(gen_output)[0, 0, 0, 0])
assert acc == list(range(100)), "Order was not keep in GeneratorEnqueuer with threads"
enqueuer.stop()
示例15
def test_ordered_enqueuer_threads_not_ordered():
enqueuer = OrderedEnqueuer(DummySequence([3, 200, 200, 3]),
use_multiprocessing=False,
shuffle=True)
enqueuer.start(3, 10)
gen_output = enqueuer.get()
acc = []
for i in range(100):
acc.append(next(gen_output)[0, 0, 0, 0])
assert acc != list(range(100)), "Order was not keep in GeneratorEnqueuer with threads"
enqueuer.stop()
示例16
def test_on_epoch_end_processes():
enqueuer = OrderedEnqueuer(DummySequence([3, 200, 200, 3]), use_multiprocessing=True)
enqueuer.start(3, 10)
gen_output = enqueuer.get()
acc = []
for i in range(200):
acc.append(next(gen_output)[0, 0, 0, 0])
assert acc[100:] == list([k * 5 for k in range(100)]), "Order was not keep in GeneratorEnqueuer with processes"
enqueuer.stop()
示例17
def test_on_epoch_end_threads():
enqueuer = OrderedEnqueuer(DummySequence([3, 200, 200, 3]), use_multiprocessing=False)
enqueuer.start(3, 10)
gen_output = enqueuer.get()
acc = []
for i in range(100):
acc.append(next(gen_output)[0, 0, 0, 0])
acc = []
for i in range(100):
acc.append(next(gen_output)[0, 0, 0, 0])
assert acc == list([k * 5 for k in range(100)]), "Order was not keep in GeneratorEnqueuer with processes"
enqueuer.stop()
示例18
def test_finite_generator_enqueuer_threads():
enqueuer = GeneratorEnqueuer(create_finite_generator_from_sequence_threads(
DummySequence([3, 200, 200, 3])), use_multiprocessing=False)
enqueuer.start(3, 10)
gen_output = enqueuer.get()
acc = []
for output in gen_output:
acc.append(int(output[0, 0, 0, 0]))
assert set(acc) == set(range(100)), "Output is not the same"
enqueuer.stop()
示例19
def test_finite_generator_enqueuer_processes():
enqueuer = GeneratorEnqueuer(create_finite_generator_from_sequence_pcs(
DummySequence([3, 200, 200, 3])), use_multiprocessing=True)
enqueuer.start(3, 10)
gen_output = enqueuer.get()
acc = []
for output in gen_output:
acc.append(int(output[0, 0, 0, 0]))
assert acc != list(range(100)), "Order was keep in GeneratorEnqueuer with processes"
enqueuer.stop()
示例20
def test_generator_enqueuer_threads():
enqueuer = GeneratorEnqueuer(create_generator_from_sequence_threads(
DummySequence([3, 200, 200, 3])), use_multiprocessing=False)
enqueuer.start(3, 10)
gen_output = enqueuer.get()
acc = []
for i in range(100):
acc.append(int(next(gen_output)[0, 0, 0, 0]))
"""
Not comparing the order since it is not guaranteed.
It may get ordered, but not a lot, one thread can take the GIL before he was supposed to.
"""
assert len(set(acc) - set(range(100))) == 0, "Output is not the same"
enqueuer.stop()
示例21
def test_generator_enqueuer_fail_threads():
enqueuer = GeneratorEnqueuer(create_generator_from_sequence_threads(
FaultSequence()), use_multiprocessing=False)
enqueuer.start(3, 10)
gen_output = enqueuer.get()
with pytest.raises(IndexError):
next(gen_output)
示例22
def test_generator_enqueuer_fail_processes():
enqueuer = GeneratorEnqueuer(create_generator_from_sequence_pcs(
FaultSequence()), use_multiprocessing=True)
enqueuer.start(3, 10)
gen_output = enqueuer.get()
with pytest.raises(IndexError):
next(gen_output)
示例23
def test_ordered_enqueuer_threads():
enqueuer = OrderedEnqueuer(DummySequence([3, 200, 200, 3]), use_multiprocessing=False)
enqueuer.start(3, 10)
gen_output = enqueuer.get()
acc = []
for i in range(100):
acc.append(next(gen_output)[0, 0, 0, 0])
assert acc == list(range(100)), "Order was not keep in GeneratorEnqueuer with threads"
enqueuer.stop()
示例24
def test_ordered_enqueuer_threads_not_ordered():
enqueuer = OrderedEnqueuer(DummySequence([3, 200, 200, 3]),
use_multiprocessing=False,
shuffle=True)
enqueuer.start(3, 10)
gen_output = enqueuer.get()
acc = []
for i in range(100):
acc.append(next(gen_output)[0, 0, 0, 0])
assert acc != list(range(100)), "Order was not keep in GeneratorEnqueuer with threads"
enqueuer.stop()
示例25
def test_ordered_enqueuer_processes():
enqueuer = OrderedEnqueuer(DummySequence([3, 200, 200, 3]), use_multiprocessing=True)
enqueuer.start(3, 10)
gen_output = enqueuer.get()
acc = []
for i in range(100):
acc.append(next(gen_output)[0, 0, 0, 0])
assert acc == list(range(100)), "Order was not keep in GeneratorEnqueuer with processes"
enqueuer.stop()
示例26
def test_on_epoch_end_threads():
enqueuer = OrderedEnqueuer(DummySequence([3, 200, 200, 3]), use_multiprocessing=False)
enqueuer.start(3, 10)
gen_output = enqueuer.get()
acc = []
for i in range(100):
acc.append(next(gen_output)[0, 0, 0, 0])
acc = []
for i in range(100):
acc.append(next(gen_output)[0, 0, 0, 0])
assert acc == list([k * 5 for k in range(100)]), "Order was not keep in GeneratorEnqueuer with processes"
enqueuer.stop()
示例27
def test_finite_generator_enqueuer_threads():
enqueuer = GeneratorEnqueuer(create_finite_generator_from_sequence_threads(
DummySequence([3, 200, 200, 3])), use_multiprocessing=False)
enqueuer.start(3, 10)
gen_output = enqueuer.get()
acc = []
for output in gen_output:
acc.append(int(output[0, 0, 0, 0]))
assert set(acc) == set(range(100)), "Output is not the same"
enqueuer.stop()
示例28
def test_finite_generator_enqueuer_processes():
enqueuer = GeneratorEnqueuer(create_finite_generator_from_sequence_pcs(
DummySequence([3, 200, 200, 3])), use_multiprocessing=True)
enqueuer.start(3, 10)
gen_output = enqueuer.get()
acc = []
for output in gen_output:
acc.append(int(output[0, 0, 0, 0]))
assert acc != list(range(100)), "Order was keep in GeneratorEnqueuer with processes"
enqueuer.stop()
示例29
def test_generator_enqueuer_threads():
enqueuer = GeneratorEnqueuer(create_generator_from_sequence_threads(
DummySequence([3, 200, 200, 3])), use_multiprocessing=False)
enqueuer.start(3, 10)
gen_output = enqueuer.get()
acc = []
for i in range(100):
acc.append(int(next(gen_output)[0, 0, 0, 0]))
"""
Not comparing the order since it is not guaranteed.
It may get ordered, but not a lot, one thread can take the GIL before he was supposed to.
"""
assert len(set(acc) - set(range(100))) == 0, "Output is not the same"
enqueuer.stop()
示例30
def test_generator_enqueuer_processes():
enqueuer = GeneratorEnqueuer(create_generator_from_sequence_pcs(
DummySequence([3, 200, 200, 3])), use_multiprocessing=True)
enqueuer.start(3, 10)
gen_output = enqueuer.get()
acc = []
for i in range(100):
acc.append(int(next(gen_output)[0, 0, 0, 0]))
assert acc != list(range(100)), "Order was keep in GeneratorEnqueuer with processes"
enqueuer.stop()
