python_multiprocessing
Différences
Ci-dessous, les différences entre deux révisions de la page.
Prochaine révision | Révision précédenteProchaine révisionLes deux révisions suivantes | ||
python_multiprocessing [2021/08/12 13:30] – créée serge | python_multiprocessing [2022/06/21 13:40] – [Mon exemple à moi archi simple] serge | ||
---|---|---|---|
Ligne 1: | Ligne 1: | ||
====== Python: Multiprocessing et GIL ====== | ====== Python: Multiprocessing et GIL ====== | ||
+ | =====Python est lent et n'est pas multiprocessus===== | ||
+ | Balivernes ... billevesées ... \\ | ||
+ | Toutes les Intelligences Artificielles ont une façade en python ... Dja dja (ah bon!)\\ | ||
+ | L' | ||
- | =====Python est lent et n'est pas multiprocess===== | + | =====GIL===== |
+ | * **https:// | ||
+ | Le module threading permet de lancer des tâches en parallèles, | ||
- | Ballivernes ... billevesées ... \\ | + | Le GIL (Global Interpreteur Lock) évite de surcharger un thread. |
- | Toutes les Intelligences Artificielles ont une façade en python ... Dja daja (ah bon!)\\ | + | |
- | L' | + | |
- | + | ||
- | =====Ressources sur le module multiprocessing===== | + | |
+ | Le module multiprocessing utilise réellement un cœur pour chaque tâche, et il est possible que chaque tâche communique avec une autre.\\ | ||
+ | Par contre, un processus n' | ||
+ | =====Ressources sur le module multiprocessing===== | ||
* http:// | * http:// | ||
* https:// | * https:// | ||
Ligne 16: | Ligne 21: | ||
* https:// | * https:// | ||
- | Mais comme souvent, les tutos sont hyper-compliqués, | + | Mais comme souvent, les tutos sont hyper-compliqués, |
- | ====Remarques==== | + | ===Remarques=== |
Certains modules, tel que numpy, sont déjà multiprocess. | Certains modules, tel que numpy, sont déjà multiprocess. | ||
- | =====Exemples===== | + | =====Exemples |
Je vous propose des exemples simples: | Je vous propose des exemples simples: | ||
+ | |||
+ | <file python multiprocessing_example_1.py> | ||
+ | """ | ||
+ | Exemple d' | ||
+ | |||
+ | documentation | ||
+ | https:// | ||
+ | |||
+ | Exemple de 2 processus qui tournent en parallèle sans communiquer. | ||
+ | """ | ||
+ | |||
+ | |||
+ | import os | ||
+ | from multiprocessing import Process | ||
+ | from time import sleep | ||
+ | |||
+ | def first_processus(): | ||
+ | print(' | ||
+ | a = 0 | ||
+ | while 1: | ||
+ | sleep(1) | ||
+ | a += 1 | ||
+ | print(" | ||
+ | |||
+ | def second_processus(): | ||
+ | print(' | ||
+ | b = 0 | ||
+ | while 1: | ||
+ | sleep(2) | ||
+ | b += 5 | ||
+ | print(" | ||
+ | |||
+ | def main(): | ||
+ | p = Process(target=second_processus) | ||
+ | p.start() | ||
+ | first_processus() | ||
+ | |||
+ | if __name__ == ' | ||
+ | main() | ||
+ | </ | ||
+ | |||
+ | <file python multiprocessing_example_2.py> | ||
+ | """ | ||
+ | |||
+ | Exemple de 2 processus qui tournent en parallèle et échangent une variable avec Pipe. | ||
+ | https:// | ||
+ | |||
+ | The Pipe() function returns a pair of connection objects connected | ||
+ | by a pipe which by default is duplex (two-way). | ||
+ | """ | ||
+ | |||
+ | |||
+ | import os | ||
+ | from multiprocessing import Process, Pipe | ||
+ | |||
+ | |||
+ | def test(n): | ||
+ | """ | ||
+ | for i in range(n): | ||
+ | m = (i ** 1.89999) ** 0.8974125 | ||
+ | |||
+ | |||
+ | def first_processus(conn): | ||
+ | print(' | ||
+ | a = 0 | ||
+ | while 1: | ||
+ | test(1000000) | ||
+ | a += 1 | ||
+ | print(" | ||
+ | b = conn.recv() | ||
+ | print(" | ||
+ | a += b[1] | ||
+ | |||
+ | def second_processus(conn): | ||
+ | print(' | ||
+ | b = 0 | ||
+ | while 1: | ||
+ | test(2000000) | ||
+ | b += 5 | ||
+ | print(" | ||
+ | conn.send([' | ||
+ | |||
+ | |||
+ | def main(): | ||
+ | """ | ||
+ | https:// | ||
+ | multiprocessing.Pipe([duplex]) | ||
+ | If duplex is True (the default) then the pipe is bidirectional. If duplex is | ||
+ | False then the pipe is unidirectional: | ||
+ | messages and conn2 can only be used for sending messages. | ||
+ | """ | ||
+ | parent_conn, | ||
+ | p = Process(target=second_processus, | ||
+ | p.start() | ||
+ | first_processus(parent_conn) | ||
+ | |||
+ | |||
+ | if __name__ == ' | ||
+ | main() | ||
+ | </ | ||
+ | |||
+ | =====Multiprocess dans kivy===== | ||
+ | Une application kivy qui excécute pyrealsense et coral dans un Process avec Pipe marche impeccable. | ||
+ | |||
+ | * **[[https:// | ||
+ | * Inferface graphique réalisée avec Kivy\\ | ||
+ | * Suivi des personnes devant la caméra | ||
+ | |||
+ | =====multiprocessing.shared_memory ===== | ||
+ | ====Ressources==== | ||
+ | * **[[https:// | ||
+ | |||
+ | ====Mon exemple à moi archi simple==== | ||
+ | Je n'ai pas besoin de montrer que je suis très fort pour me faire embaucher chez GAFAM. | ||
+ | |||
+ | <code python> | ||
+ | from time import time, sleep | ||
+ | import random | ||
+ | from multiprocessing import Process, shared_memory | ||
+ | from multiprocessing.sharedctypes import Value | ||
+ | |||
+ | class SharedMemory: | ||
+ | def __init__(self): | ||
+ | self.val = Value(" | ||
+ | print(self.val, | ||
+ | my_proc = Process(target=another_process, | ||
+ | my_proc.start() | ||
+ | |||
+ | def shared_memory_master(self): | ||
+ | t = time() | ||
+ | while time() - t < 4: | ||
+ | print(f" | ||
+ | sleep(0.1) | ||
+ | |||
+ | def another_process(val): | ||
+ | t = time() | ||
+ | while time() - t < 3: | ||
+ | n = random.randint(-1000, | ||
+ | print(f" | ||
+ | val.value = n | ||
+ | sleep(0.3) | ||
+ | |||
+ | if __name__ == " | ||
+ | sh = SharedMemory() | ||
+ | sh.shared_memory_master() | ||
+ | </ | ||
+ | ====Exemple d' | ||
+ | |||
+ | <code python> | ||
+ | from multiprocessing import shared_memory | ||
+ | |||
+ | shm_a = shared_memory.SharedMemory(create=True, | ||
+ | m = shm_a.buf | ||
+ | print(' | ||
+ | |||
+ | buffer = shm_a.buf | ||
+ | print(' | ||
+ | |||
+ | # Modify multiple at once | ||
+ | buffer[:4] = bytearray([22, | ||
+ | # Modify single byte at a time | ||
+ | buffer[4] = 100 | ||
+ | |||
+ | # Attach to an existing shared memory block | ||
+ | shm_b = shared_memory.SharedMemory(shm_a.name) | ||
+ | |||
+ | import array | ||
+ | |||
+ | # Copy the data into a new array.array | ||
+ | ar = array.array(' | ||
+ | print(' | ||
+ | |||
+ | # Modify via shm_b using bytes | ||
+ | shm_b.buf[: | ||
+ | |||
+ | # Access via shm_a | ||
+ | print(' | ||
+ | |||
+ | # Close each SharedMemory instance | ||
+ | shm_b.close() | ||
+ | shm_a.close() | ||
+ | |||
+ | # Call unlink only once to release the shared memory | ||
+ | shm_a.unlink() | ||
+ | </ | ||
+ | |||
+ | ====Exemple de partage de numpy array==== | ||
+ | <code python> | ||
+ | from multiprocessing import Process | ||
+ | from multiprocessing.managers import SharedMemoryManager | ||
+ | from multiprocessing.shared_memory import SharedMemory | ||
+ | |||
+ | import numpy as np | ||
+ | |||
+ | |||
+ | def test(shared_mem: | ||
+ | a = np.frombuffer(shared_mem.buf, | ||
+ | a[0] = -a[0] | ||
+ | |||
+ | |||
+ | if __name__ == " | ||
+ | # Create the array | ||
+ | N = int(10) | ||
+ | unshared_arr = np.random.rand(N) | ||
+ | DTYPE = unshared_arr.dtype | ||
+ | with SharedMemoryManager() as smm: | ||
+ | shared_mem = smm.SharedMemory(size=unshared_arr.nbytes) | ||
+ | arr = np.frombuffer(shared_mem.buf, | ||
+ | arr[:] = unshared_arr | ||
+ | print(" | ||
+ | |||
+ | # Create, start, and finish the child processes | ||
+ | p = Process(target=test, | ||
+ | p.start() | ||
+ | p.join() | ||
+ | |||
+ | # Printing out the changed values | ||
+ | print(" | ||
+ | </ | ||
+ | |||
+ | |||
+ | {{tag> kivy python sb }} |
python_multiprocessing.txt · Dernière modification : 2022/06/21 13:42 de serge