Python分布式计算(有效)

本文介绍了Python分布式计算(有效)的处理方法，对大家解决问题具有一定的参考价值，需要的朋友们下面随着小编来一起学习吧！ 问题描述

我正在使用旧线程发布新代码，以尝试解决相同的问题. 什么是安全的泡菜? 这?

I'm using an old thread to post new code which attempts to solve the same problem. What constitutes a secure pickle? this?

sock.py

from socket import socket from socket import AF_INET from socket import SOCK_STREAM from socket import gethostbyname from socket import gethostname class SocketServer: def __init__(self, port): self.sock = socket(AF_INET, SOCK_STREAM) self.port = port def listen(self, data): self.sock.bind(("127.0.0.1", self.port)) self.sock.listen(len(data)) while data: s = self.sock.accept()[0] siz, dat = data.pop() s.send(siz) s.send(dat) s.close() class Socket: def __init__(self, host, port): self.sock = socket(AF_INET, SOCK_STREAM) self.sock.connect((host, port)) def recv(self, size): return self.sock.recv(size)

pack.py

#stackoverflow/questions/6234586/we-need-to-pickle-any-sort-of-callable from marshal import dumps as marshal_dumps from pickle import dumps as pickle_dumps from struct import pack as struct_pack class packer: def __init__(self): self.f = [] def pack(self, what): if type(what) is type(lambda:None): self.f = [] self.f.append(marshal_dumps(what.func_code)) self.f.append(pickle_dumps(what.func_name)) self.f.append(pickle_dumps(what.func_defaults)) self.f.append(pickle_dumps(what.func_closure)) self.f = pickle_dumps(self.f) return (struct_pack('Q', len(self.f)), self.f)

unpack.py

unpack.py

from types import FunctionType from pickle import loads as pickle_loads from marshal import loads as marshal_loads from struct import unpack as struct_unpack from struct import calcsize #stackoverflow/questions/6234586/we-need-to-pickle-any-sort-of-callable class unpacker: def __init__(self): self.f = [] self.fcompiled = lambda:None self.sizeofsize = calcsize('Q') def unpack(self, sock): size = struct_unpack('Q', sock.recv(self.sizeofsize))[0] self.f = pickle_loads(sock.recv(size)) a = marshal_loads(self.f[0]) b = globals() ## c = pickle_loads(self.f[1]) d = pickle_loads(self.f[2]) e = pickle_loads(self.f[3]) self.fcompiled = FunctionType(a, b, c, d, e) return self.fcompiled

test.py

from unpack import unpacker from pack import packer from sock import SocketServer from sock import Socket from threading import Thread from time import sleep count = 2 port = 4446 def f(): print 42 def server(): ss = SocketServer(port) pack = packer() functions = [pack.pack(f) for nothing in range(count)] ss.listen(functions) if __name__ == "__main__": Thread(target=server).start() sleep(1) unpack = unpacker() for nothing in range(count): print unpack.unpack(Socket("127.0.0.1", port))

输出:

<function f at 0x12917d0> <function f at 0x12915f0>

当您的泡菜损坏时，会引发

推荐答案

ValueError: insecure string pickle.您确定要在一个sock.recv()(unpack.py)中收到整个腌制的对象吗?

ValueError: insecure string pickle is raised when your pickle is corrupted. Are you sure you are receiving the entire pickled object in one sock.recv() (unpack.py)?

为避免出现任何大小问题，您都可以这样做(您的Socket类必须支持使用带有缓冲区大小参数的recv调用(即

to avoid this for any size you could do (your Socket class would have to support recv to be called with an buffer size argument (i.e

class Socket: def recv(self, bufsize): return self.sock.recv(bufsize)

)):

import struct struct.pack('Q', len(pickled_list)) # Send it, and then send the pickled list.

在接收程序中:

import struct length = struct.unpack('Q', sock.recv(struct.calcsize('Q')))[0] pickled_list = sock.recv(length)

'Q'是unsigned long long.有关其他结构的信息，请参见结构模块文档

'Q' is an unsigned long long. For other struct things see the struct module documentation