Edit: multi-socket_action-select.py
#! /usr/bin/env python # -*- coding: utf-8 -*- # vi:ts=4:et # Retrieves a single URL using the CurlMulti.socket_action calls, using # select as the I/O polling mechanism: # # First, create a Multi object, and set socket and timer callbacks on it. # Observed side effect: this causes the timer callback to be immediately # invoked with the zero value for the timeout. # # The timer callback is very simple - it stores the timeout value passed # into it in the global state for future use by the select calls that # we will be making. # # The socket callback is more complicated. Its job is to add and remove # socket handles to/from the data structure that we use for waiting for # activity on them. The callback is invoked with a socket handle and the # needed action (add for reading, add for writing or remove). # Since this script utilizes the select call for waiting for activity, # the socket callback updates the list of sockets which we should be # polling for readability and the list that we should be polling for # writability, which are then passed to the select call (and both of the # sets are passed as the sockets to wait for errors/exceptions on). # # Next, create a Curl object (mapping to a libcurl easy handle), set the URL # we are going to retrieve as well as any transfer options. This script sets # the timeout to 5 seconds to be able to test failing transfers easily. # # Add the Curl object to the Multi object. # # Invoke Multi.socket_action to start the retrieval operation. # Observed side effect: this causes the timer callback to be invoked # with a greater than zero value for the timeout. # # By now we should have initialized our own state, which this script does # prior to invoking any libcurl functions. Importantly, the state includes # the timeout value that was communicated to us by libcurl. # # Run a loop which waits for activity on any of the sockets used by libcurl. # The sockets are set that the socket callback has produced as of the # present moment; the timeout is the most recent timeout value received by # the timer callback. # # Importantly, the loop should not simply sleep for the entire # timeout interval, as that would cause the transfer to take a very long time. # It is *required* to use something like a select call to wait for activity # on any of the sockets currently active for *up to* the timeout value. # # The loop terminates when the number of active transfers (handles in libcurl # parlance) reaches zero. This number is provided by each socket_action # call, which is why each call (even the ones that are called due to # timeout being reached, as opposed to any socket activity) must update # the number of running handles. # # After the loop terminates, clean up everything: remove the easy object from # the multi object, close the easy object, close the multi object. import sys, select import pycurl from io import BytesIO if len(sys.argv) > 1: url = sys.argv[1] else: url = 'https://www.python.org' state = { 'rlist': [], 'wlist': [], 'running': None, 'timeout': None, 'result': None, # If the transfer failed, code and msg will be filled in. 'code': None, 'msg': None, } def socket_fn(what, sock_fd, multi, socketp): if what == pycurl.POLL_IN or what == pycurl.POLL_INOUT: state['rlist'].append(sock_fd) elif what == pycurl.POLL_OUT or what == pycurl.POLL_INOUT: state['wlist'].append(sock_fd) elif what == pycurl.POLL_REMOVE: if sock_fd in state['rlist']: state['rlist'].remove(sock_fd) if sock_fd in state['wlist']: state['wlist'].remove(sock_fd) else: raise Exception("Unknown value of what: %s" % what) def work(timeout): rready, wready, xready = select.select( state['rlist'], state['wlist'], set(state['rlist']) | set(state['wlist']), timeout) if len(rready) == 0 and len(wready) == 0 and len(xready) == 0: # The number of running handles must be updated after each # call to socket_action, which includes those with the SOCKET_TIMEOUT # argument (otherwise e.g. a transfer which failed due to # exceeding the connection timeout would hang). _, running = multi.socket_action(pycurl.SOCKET_TIMEOUT, 0) else: for sock_fd in rready: # socket_action returns a tuple whose first element is always the # CURLE_OK value (0), ignore it and use the second element only. _, running = multi.socket_action(sock_fd, pycurl.CSELECT_IN) for sock_fd in wready: _, running = multi.socket_action(sock_fd, pycurl.CSELECT_OUT) for sock_fd in xready: _, running = multi.socket_action(sock_fd, pycurl.CSELECT_ERR) # Since we are only performing a single transfer, we could call # Multi.info_read after the I/O loop terminates. # In practice, you would probably use socket_action with multiple # transfers, and you may want to be notified about transfer completion # as soon as the result is available. if state['running'] is not None and running != state['running']: # Some handle has completed. # # Note that socket_action was potentially called multiple times # in this function (e.g. if both a read handle became ready and a # different write handle became ready), therefore it is possible # that multiple handles have completed. In this particular script # we are only performing a single transfer (one # Curl object / easy handle), therefore only one transfer can ever # possibly complete. qmsg, successes, failures = multi.info_read() # We should have retrieved all of the available statuses, leaving # none in the queue. assert qmsg == 0 # We have only one transfer. assert len(successes) == 1 and len(failures) == 0 or \ len(successes) == 0 and len(failures) == 1 if successes: state['result'] = True if failures: state['result'] = False # The failures array contains tuples of # (easy object, CURLE code, error message). _easy, state['code'], state['msg'] = failures[0] state['running'] = running def timer_fn(timeout_ms): if timeout_ms < 0: # libcurl passes a negative timeout value when no further # calls should be made. state['timeout'] = None else: state['timeout'] = timeout_ms / 1000.0 multi = pycurl.CurlMulti() multi.setopt(pycurl.M_SOCKETFUNCTION, socket_fn) multi.setopt(pycurl.M_TIMERFUNCTION, timer_fn) easy = pycurl.Curl() easy.setopt(pycurl.URL, url) # Uncomment to see what libcurl is doing throughout the transfer. #easy.setopt(pycurl.VERBOSE, 1) easy.setopt(pycurl.CONNECTTIMEOUT, 5) easy.setopt(pycurl.LOW_SPEED_TIME, 5) easy.setopt(pycurl.LOW_SPEED_LIMIT, 1) _io = BytesIO() easy.setopt(pycurl.WRITEDATA, _io) multi.add_handle(easy) handles = multi.socket_action(pycurl.SOCKET_TIMEOUT, 0) # This should invoke the timer function with a timeout value. while True: if state['running'] == 0: break else: # By the time we get here, timer function should have been already # invoked at least once so that we have a libcurl-supplied # timeout value. But in case this hasn't happened, default the timeout # to 1 second. timeout = state['timeout'] if timeout is None: raise Exception('Need to poll for I/O but the timeout is not set!') work(timeout) multi.remove_handle(easy) easy.close() multi.close() # Uncomment to print the retrieved contents. #print(_io.getvalue().decode()) if state['result'] is None: raise Exception('Script finished without a result!') if state['result']: print('Transfer successful, retrieved %d bytes' % len(_io.getvalue())) else: print('Transfer failed with code %d: %s' % (state['code'], state['msg']))
Cancel