Найти - Пользователи
Полная версия: twisted client
Начало » Python для экспертов » twisted client
1
plusplus
Разбираюсь с twisted-ом. Читаю книгу “Введение в асинхронное программирование и Twisted”. Там есть такой пример клиента:
# This is the Twisted Get Poetry Now! client, version 4.0
import optparse, sys
from twisted.internet import defer
from twisted.internet.protocol import Protocol, ClientFactory
def parse_args():
    usage = """usage: %prog [options] [hostname]:port ...
This is the Get Poetry Now! client, Twisted version 4.0
Run it like this:
  python get-poetry.py port1 port2 port3 ...
If you are in the base directory of the twisted-intro package,
you could run it like this:
  python twisted-client-4/get-poetry.py 10001 10002 10003
to grab poetry from servers on ports 10001, 10002, and 10003.
Of course, there need to be servers listening on those ports
for that to work.
"""
    parser = optparse.OptionParser(usage)
    _, addresses = parser.parse_args()
    if not addresses:
        print parser.format_help()
        parser.exit()
    def parse_address(addr):
        if ':' not in addr:
            host = '127.0.0.1'
            port = addr
        else:
            host, port = addr.split(':', 1)
        if not port.isdigit():
            parser.error('Ports must be integers.')
        return host, int(port)
    return map(parse_address, addresses)
class PoetryProtocol(Protocol):
    poem = ''
    def dataReceived(self, data):
        self.poem += data
    def connectionLost(self, reason):
        self.poemReceived(self.poem)
    def poemReceived(self, poem):
        self.factory.poem_finished(poem)
class PoetryClientFactory(ClientFactory):
    protocol = PoetryProtocol
    def __init__(self, deferred):
        self.deferred = deferred
    def poem_finished(self, poem):
        if self.deferred is not None:
            d, self.deferred = self.deferred, None
            d.callback(poem)
    def clientConnectionFailed(self, connector, reason):
        if self.deferred is not None:
            d, self.deferred = self.deferred, None
            d.errback(reason)
def get_poetry(host, port):
    """
    Download a poem from the given host and port. This function
    returns a Deferred which will be fired with the complete text of
    the poem or a Failure if the poem could not be downloaded.
    """
    d = defer.Deferred()
    from twisted.internet import reactor
    factory = PoetryClientFactory(d)
    reactor.connectTCP(host, port, factory)
    return d
def poetry_main():
    addresses = parse_args()
    from twisted.internet import reactor
    poems = []
    errors = []
    def got_poem(poem):
        poems.append(poem)
    def poem_failed(err):
        print >>sys.stderr, 'Poem failed:', err
        errors.append(err)
    def poem_done(_):
        if len(poems) + len(errors) == len(addresses):
            reactor.stop()
    for address in addresses:
        host, port = address
        d = get_poetry(host, port)
        d.addCallbacks(got_poem, poem_failed)
        d.addBoth(poem_done)
    reactor.run()
    for poem in poems:
        print poem
if __name__ == '__main__':
    poetry_main()

Вопрос в следующем, а как мне поступить, если бы у меня было очень много адресов, скажем несколько тысяч? Как мне организовать для них, что-то вроде пула потоков?
o7412369815963
Можно ограничить одновременную обработку (например до 50), и по завершении активных запускать ещё задач.

Как вариант можно сделать ф-ию запуска
def start_task():
        if not addresses:
            return
        host, port = addresses.pop()
        d = get_poetry(host, port)
        d.addCallbacks(got_poem, poem_failed)
        d.addBoth(poem_done)
И запихать её в poem_done, и в начальный цикл, который ограничить до 50.
В poem_done делать reactor.stop() если нет активных задач.
plusplus
Нашел ответ здесь http://technicae.cogitat.io/2008/06/async-batching-with-twisted-walkthrough.html
This is a "lo-fi" version of our main content. To view the full version with more information, formatting and images, please click here.
Powered by DjangoBB