From 2c7521910bc571b27c76d64d2e005d30657f958f Mon Sep 17 00:00:00 2001 From: Mitar Date: Thu, 3 Nov 2011 02:25:54 +0100 Subject: [PATCH 01/14] Allow polling on a not (yet) existing channel. --- README.rst | 1 + bin/hbpushd | 5 +++-- hbpush/pubsub/subscriber.py | 6 +++++- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index 423bd98..a5dab28 100644 --- a/README.rst +++ b/README.rst @@ -161,6 +161,7 @@ A location has a ``type`` of either ``publisher`` or ``subscriber``. It supports - ``url``: the complete URL pattern to use for this location, eg: ``/channel/(\d+)/publish/``. Not you should have only one capture group, that must represent the channel id. This settings has precedence over ``prefix`` (not set by default) - ``polling`` (subscriber only): ``interval`` or ``long``, see the protocol_ for more information (default to ``long``) - ``create_on_post`` (publisher only): if set to ``false``, you will need to create a channel with a PUT request first before POSTing any data to it (default to ``true``) +- ``create_on_get`` (subscriber only): if set to ``false``, a channel has to be first created before the first GET request (default to ``true``) For info, the default configuration looks like this:: diff --git a/bin/hbpushd b/bin/hbpushd index 3cf9420..8173400 100755 --- a/bin/hbpushd +++ b/bin/hbpushd @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python ## DEFAULT CONFIGURATION ## default_store= { @@ -15,6 +15,7 @@ default_store= { default_location = { 'subscriber': { 'polling': 'long', + 'create_on_get': True, 'store': 'default', }, 'publisher': { @@ -118,7 +119,7 @@ def make_location(loc_dict, stores={}): else: raise InvalidConfigurationError('Invalid location type `%s`' % loc_type) - url = loc_conf.pop('url', loc_conf.pop('prefix')+'(.+)') + url = loc_conf.pop('url', loc_conf.pop('prefix', '')+'(.+)') store_id = loc_conf.pop('store') kwargs = {'registry': stores[store_id]['registry']} kwargs.update(loc_conf) diff --git a/hbpush/pubsub/subscriber.py b/hbpush/pubsub/subscriber.py index 6bd0e25..cec3d1b 100644 --- a/hbpush/pubsub/subscriber.py +++ b/hbpush/pubsub/subscriber.py @@ -6,6 +6,10 @@ from functools import partial class Subscriber(PubSubHandler): + def __init__(self, *args, **kwargs): + self.create_on_get = kwargs.pop('create_on_get', True) + super(Subscriber, self).__init__(*args, **kwargs) + @asynchronous def get(self, channel_id): try: @@ -14,7 +18,7 @@ def get(self, channel_id): except: raise HTTPError(400) - self.registry.get(channel_id, + getattr(self.registry, 'get_or_create' if self.create_on_get else 'get')(channel_id, callback=self.async_callback(partial(self._process_channel, last_modified, etag)), errback=self.errback) From 0a86394c874877a3d1007ffde4183485f174e91e Mon Sep 17 00:00:00 2001 From: Mitar Date: Thu, 3 Nov 2011 04:06:43 +0100 Subject: [PATCH 02/14] Added messages expiring for memory store. --- README.rst | 6 +++++- bin/hbpushd | 3 +++ hbpush/channel.py | 15 +++++++------- hbpush/message.py | 3 +++ hbpush/store/__init__.py | 3 +++ hbpush/store/memory.py | 43 ++++++++++++++++++++++++++++++++++++---- hbpush/store/redis.py | 3 ++- 7 files changed, 63 insertions(+), 13 deletions(-) diff --git a/README.rst b/README.rst index a5dab28..4cd80b3 100644 --- a/README.rst +++ b/README.rst @@ -106,7 +106,11 @@ of stores, ``memory`` and ``redis``. Each of these stores has specific options. - ``key_prefix``: a string prepended to a channel identifier to make a redis key. Use this to avoid key collision when you're using your redis server for other stuff. -Memory stores haven't any specific options (yet). +For memory stores: + +- ``min_messages``: the minimum number of messages to store per channel +- ``max_messages``: the maximum number of messages to store per channel +- ``message_timeout``: the length of time a message may be queued before it is expired Here is an example of how to specify the store (YAML):: diff --git a/bin/hbpushd b/bin/hbpushd index 8173400..8c0540c 100755 --- a/bin/hbpushd +++ b/bin/hbpushd @@ -9,6 +9,9 @@ default_store= { 'database': 0, }, 'memory': { + 'min_messages': 0, + 'max_messages': 0, + 'message_timeout': 0, } } diff --git a/hbpush/channel.py b/hbpush/channel.py index aecf6d9..d8c6ab5 100644 --- a/hbpush/channel.py +++ b/hbpush/channel.py @@ -23,9 +23,6 @@ def __init__(self, id, store): # Empty message, we just want to keep etag and lastmodified data self.last_message = Message(0, -1) - def get_last_message(self): - return self.last_message - def send_to_subscribers(self, message): # We work on a copy to deal with reentering subscribers subs = self.subscribers.copy() @@ -70,9 +67,13 @@ def get(self, last_modified, etag, callback, errback): request_msg = Message(last_modified, etag) if request_msg < self.last_message: - self.store.get(self.id, last_modified, etag, callback=callback, errback=errback) - else: - errback(Channel.NotModified()) + try: + self.store.get(self.id, last_modified, etag, callback=callback, errback=errback) + return + except Message.Expired: + pass + + errback(Channel.NotModified()) def delete(self, callback, errback): for id, (cb, eb) in self.subscribers.items(): @@ -85,7 +86,7 @@ def delete(self, callback, errback): def make_message(self, content_type, body): if not self.sentinel: - self.sentinel = self.get_last_message() + self.sentinel = self.last_message last_modified = int(time.time()) if last_modified == self.sentinel.last_modified: diff --git a/hbpush/message.py b/hbpush/message.py index 6011c00..03047e0 100644 --- a/hbpush/message.py +++ b/hbpush/message.py @@ -16,3 +16,6 @@ class DoesNotExist(Exception): class Invalid(Exception): pass + + class Expired(Exception): + pass diff --git a/hbpush/store/__init__.py b/hbpush/store/__init__.py index cb63129..2c63f8f 100644 --- a/hbpush/store/__init__.py +++ b/hbpush/store/__init__.py @@ -1,4 +1,7 @@ class Store(object): + def __init__(self, *args, **kwargs): + pass + def get(self, channel_id, last_modified, etag, callback, errback): raise NotImplementedError("") diff --git a/hbpush/store/memory.py b/hbpush/store/memory.py index d0c301b..7c86698 100644 --- a/hbpush/store/memory.py +++ b/hbpush/store/memory.py @@ -2,16 +2,47 @@ from hbpush.message import Message from bisect import bisect +import time class MemoryStore(Store): def __init__(self, *args, **kwargs): super(MemoryStore, self).__init__(*args, **kwargs) + self.min_messages = kwargs.pop('min_messages', 0) + self.max_messages = kwargs.pop('max_messages', 0) + self.message_timeout = kwargs.pop('message_timeout', 0) self.messages = {} + self.expired_channels = {} - def get(self, channel_id, last_modified, etag, callback, errback): + def _expire_messages(self, channel_id): channel_messages = self.messages.setdefault(channel_id, []) + if not channel_messages: + if self.expired_channels.get(channel_id, False): + raise Message.Expired() + else: + return channel_messages + + if self.max_messages and len(channel_messages) > self.max_messages: + channel_messages = channel_messages[-self.max_messages:] + + if self.message_timeout: + while channel_messages and len(channel_messages) > self.min_messages: + if channel_messages[0].last_modified + self.message_timeout >= int(time.time()): + break + channel_messages = channel_messages[1:] + + self.messages[channel_id] = channel_messages + + if not self.messages[channel_id]: + self.expired_channels[channel_id] = True + raise Message.Expired() + else: + return channel_messages + + def get(self, channel_id, last_modified, etag, callback, errback): + channel_messages = self._expire_messages(channel_id) + msg = Message(last_modified, etag) try: callback(channel_messages[bisect(channel_messages, msg)]) @@ -19,21 +50,25 @@ def get(self, channel_id, last_modified, etag, callback, errback): errback(Message.DoesNotExist()) def get_last(self, channel_id, callback, errback): - channel_messages = self.messages.setdefault(channel_id, []) + channel_messages = self._expire_messages(channel_id) - if len(channel_messages): + if channel_messages: callback(channel_messages[-1]) else: errback(Message.DoesNotExist()) def post(self, channel_id, message, callback, errback): - self.messages.setdefault(channel_id, []).append(message) + channel_messages = self.messages.setdefault(channel_id, []) + self.messages[channel_id].append(message) + self.expired_channels[channel_id] = False callback(message) def flush(self, channel_id, callback, errback): del self.messages[channel_id] + del self.expired_channels[channel_id] callback(True) def flushall(self, callback, errback): self.messages = {} + self.expired_channels = {} callback(True) diff --git a/hbpush/store/redis.py b/hbpush/store/redis.py index 7eba1f9..c968694 100644 --- a/hbpush/store/redis.py +++ b/hbpush/store/redis.py @@ -9,7 +9,8 @@ class RedisStore(Store): - def __init__(self, **kwargs): + def __init__(self, *args, **kwargs): + super(RedisStore, self).__init__(*args, **kwargs) self.key_prefix = kwargs.pop('key_prefix', '') self.client = Client(**kwargs) self.client.connect() From d04f92118bc6f7ebf8b82d5d7f8f74f79e6a5e02 Mon Sep 17 00:00:00 2001 From: Mitar Date: Thu, 3 Nov 2011 05:15:16 +0100 Subject: [PATCH 03/14] Reverted the default for create_on_get. Updated setup.py. --- README.rst | 2 +- bin/hbpushd | 2 +- hbpush/pubsub/subscriber.py | 2 +- hbpush/store/memory.py | 3 +-- setup.py | 7 +++---- tests/mocks.py | 6 +++++- 6 files changed, 12 insertions(+), 10 deletions(-) diff --git a/README.rst b/README.rst index 4cd80b3..8ef1c8e 100644 --- a/README.rst +++ b/README.rst @@ -165,7 +165,7 @@ A location has a ``type`` of either ``publisher`` or ``subscriber``. It supports - ``url``: the complete URL pattern to use for this location, eg: ``/channel/(\d+)/publish/``. Not you should have only one capture group, that must represent the channel id. This settings has precedence over ``prefix`` (not set by default) - ``polling`` (subscriber only): ``interval`` or ``long``, see the protocol_ for more information (default to ``long``) - ``create_on_post`` (publisher only): if set to ``false``, you will need to create a channel with a PUT request first before POSTing any data to it (default to ``true``) -- ``create_on_get`` (subscriber only): if set to ``false``, a channel has to be first created before the first GET request (default to ``true``) +- ``create_on_get`` (subscriber only): if set to ``true``, a non-existing channel will be automatically created at the first GET request (default to ``false``) For info, the default configuration looks like this:: diff --git a/bin/hbpushd b/bin/hbpushd index 8c0540c..5da4d87 100755 --- a/bin/hbpushd +++ b/bin/hbpushd @@ -18,7 +18,7 @@ default_store= { default_location = { 'subscriber': { 'polling': 'long', - 'create_on_get': True, + 'create_on_get': False, 'store': 'default', }, 'publisher': { diff --git a/hbpush/pubsub/subscriber.py b/hbpush/pubsub/subscriber.py index cec3d1b..289734a 100644 --- a/hbpush/pubsub/subscriber.py +++ b/hbpush/pubsub/subscriber.py @@ -7,7 +7,7 @@ class Subscriber(PubSubHandler): def __init__(self, *args, **kwargs): - self.create_on_get = kwargs.pop('create_on_get', True) + self.create_on_get = kwargs.pop('create_on_get', False) super(Subscriber, self).__init__(*args, **kwargs) @asynchronous diff --git a/hbpush/store/memory.py b/hbpush/store/memory.py index 7c86698..f17f589 100644 --- a/hbpush/store/memory.py +++ b/hbpush/store/memory.py @@ -58,8 +58,7 @@ def get_last(self, channel_id, callback, errback): errback(Message.DoesNotExist()) def post(self, channel_id, message, callback, errback): - channel_messages = self.messages.setdefault(channel_id, []) - self.messages[channel_id].append(message) + self.messages.setdefault(channel_id, []).append(message) self.expired_channels[channel_id] = False callback(message) diff --git a/setup.py b/setup.py index ae306a7..646d56e 100644 --- a/setup.py +++ b/setup.py @@ -27,9 +27,8 @@ packages=('hbpush', 'hbpush.store', 'hbpush.utils', 'hbpush.pubsub'), scripts=('bin/hbpushd',), - dependency_links= ('http://github.com/facebook/tornado/tarball/b8271f94434208646eeec9cf33da703d97c5364e#egg=tornado-0.2', - 'http://github.com/clement/brukva/tarball/bff451511a3cc09cd52bebcf6372a59d36567827#egg=brukva-0.0.1',), + dependency_links= ('http://github.com/clement/brukva/tarball/bff451511a3cc09cd52bebcf6372a59d36567827#egg=brukva-0.0.1',), setup_requires=('nose>=0.11',), - install_requires=('PyYAML', 'brukva==0.0.1', 'tornado==0.2'), - requires=('PyYAML', 'brukva(==0.0.1)', 'tonardo(==0.2)'), + install_requires=('PyYAML', 'brukva>0.0.1', 'tornado>0.2'), + requires=('PyYAML', 'brukva(>0.0.1)', 'tonardo(>0.2)'), ) diff --git a/tests/mocks.py b/tests/mocks.py index 6da8acf..89bb896 100644 --- a/tests/mocks.py +++ b/tests/mocks.py @@ -1,6 +1,10 @@ from hbpush.pubsub.publisher import Publisher from hbpush.pubsub.subscriber import Subscriber, LongPollingSubscriber -from tornado.httpserver import HTTPHeaders, HTTPRequest +from tornado.httpserver import HTTPRequest +try: + from tornado.httpserver import HTTPHeaders +except ImportError: + from tornado.httputil import HTTPHeaders from tornado.web import HTTPError from tornado.ioloop import IOLoop From b55a8e352e9b6b28c27a3e96332e7450f208993b Mon Sep 17 00:00:00 2001 From: Mitar Date: Thu, 3 Nov 2011 05:19:51 +0100 Subject: [PATCH 04/14] Bugfix. --- hbpush/store/memory.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hbpush/store/memory.py b/hbpush/store/memory.py index f17f589..3a694ec 100644 --- a/hbpush/store/memory.py +++ b/hbpush/store/memory.py @@ -64,7 +64,8 @@ def post(self, channel_id, message, callback, errback): def flush(self, channel_id, callback, errback): del self.messages[channel_id] - del self.expired_channels[channel_id] + if channel_id in self.expired_channels: + del self.expired_channels[channel_id] callback(True) def flushall(self, callback, errback): From a929a1c486526683d05b9f1022ac80463d3460e3 Mon Sep 17 00:00:00 2001 From: Mitar Date: Thu, 3 Nov 2011 18:51:51 +0100 Subject: [PATCH 05/14] Added Cross-Origin Resource Sharing headers. --- README.rst | 1 + bin/hbpushd | 1 + hbpush/pubsub/__init__.py | 7 +++++++ hbpush/pubsub/subscriber.py | 8 ++++++++ 4 files changed, 17 insertions(+) diff --git a/README.rst b/README.rst index 8ef1c8e..515d2ed 100644 --- a/README.rst +++ b/README.rst @@ -166,6 +166,7 @@ A location has a ``type`` of either ``publisher`` or ``subscriber``. It supports - ``polling`` (subscriber only): ``interval`` or ``long``, see the protocol_ for more information (default to ``long``) - ``create_on_post`` (publisher only): if set to ``false``, you will need to create a channel with a PUT request first before POSTing any data to it (default to ``true``) - ``create_on_get`` (subscriber only): if set to ``true``, a non-existing channel will be automatically created at the first GET request (default to ``false``) +- ``allow_origin`` (subscriber only): value of ``Access-Control-Allow-Origin`` header send as defined by Cross-Origin Resource Sharing specification (default to ``*``) For info, the default configuration looks like this:: diff --git a/bin/hbpushd b/bin/hbpushd index 5da4d87..18f3b44 100755 --- a/bin/hbpushd +++ b/bin/hbpushd @@ -20,6 +20,7 @@ default_location = { 'polling': 'long', 'create_on_get': False, 'store': 'default', + 'allow_origin': '*', }, 'publisher': { 'create_on_post': True, diff --git a/hbpush/pubsub/__init__.py b/hbpush/pubsub/__init__.py index e6d6261..3b57d17 100644 --- a/hbpush/pubsub/__init__.py +++ b/hbpush/pubsub/__init__.py @@ -14,11 +14,18 @@ class PubSubHandler(RequestHandler): def __init__(self, *args, **kwargs): self.registry = kwargs.pop('registry', None) + self.allow_origin = kwargs.pop('allow_origin', '*') super(PubSubHandler, self).__init__(*args, **kwargs) def add_vary_header(self): self.set_header('Vary', 'If-Modified-Since, If-None-Match') + def add_accesscontrol_headers(self): + self.set_header('Access-Control-Allow-Origin', self.allow_origin) + self.set_header('Access-Control-Allow-Headers', 'If-Modified-Since, If-None-Match') + self.set_header('Access-Control-Expose-Headers', 'Last-Modified, Etag, Cache-Control') + self.set_header('Access-Control-Max-Age', '864000') + def _handle_request_exception(self, e): if e.__class__ in self.exception_mapping: e = HTTPError(self.exception_mapping[e.__class__], str(e)) diff --git a/hbpush/pubsub/subscriber.py b/hbpush/pubsub/subscriber.py index 289734a..75a854d 100644 --- a/hbpush/pubsub/subscriber.py +++ b/hbpush/pubsub/subscriber.py @@ -22,10 +22,18 @@ def get(self, channel_id): callback=self.async_callback(partial(self._process_channel, last_modified, etag)), errback=self.errback) + def options(self, channel_id): + self.add_accesscontrol_headers() + def _process_message(self, message): self.set_header('Etag', message.etag) + # Chrome and other WebKit-based browsers do not (yet) support Access-Control-Expose-Headers, + # but they allow access to Cache-Control so we use it to additionally store etag information there + # (This field is by standard extendable with custom tokens) + self.set_header('Cache-Control', '%s=%s' % ('etag', message.etag)) self.set_header('Last-Modified', formatdate(message.last_modified, localtime=False, usegmt=True)) self.add_vary_header() + self.add_accesscontrol_headers() self.set_header('Content-Type', message.content_type) self.write(message.body) self.finish() From 828419cfb0d76f8687d9e2d8258d04db6295cc7a Mon Sep 17 00:00:00 2001 From: Mitar Date: Fri, 11 Nov 2011 16:09:21 +0100 Subject: [PATCH 06/14] Fixed dependencies in setup.py. --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 646d56e..7aa3b85 100644 --- a/setup.py +++ b/setup.py @@ -29,6 +29,6 @@ scripts=('bin/hbpushd',), dependency_links= ('http://github.com/clement/brukva/tarball/bff451511a3cc09cd52bebcf6372a59d36567827#egg=brukva-0.0.1',), setup_requires=('nose>=0.11',), - install_requires=('PyYAML', 'brukva>0.0.1', 'tornado>0.2'), - requires=('PyYAML', 'brukva(>0.0.1)', 'tonardo(>0.2)'), + install_requires=('PyYAML', 'brukva>=0.0.1', 'tornado>0.2'), + requires=('PyYAML', 'brukva(>=0.0.1)', 'tonardo(>0.2)'), ) From f945b755a5a29d98f214408425d027b7b9d47205 Mon Sep 17 00:00:00 2001 From: Mitar Date: Mon, 9 Apr 2012 05:17:02 +0200 Subject: [PATCH 07/14] Support for allowing credentials and request headers passthrough. --- README.rst | 2 ++ bin/hbpushd | 2 ++ hbpush/channel.py | 28 +++++++++++++++++++++++----- hbpush/pubsub/__init__.py | 6 +++++- hbpush/pubsub/subscriber.py | 5 +++-- 5 files changed, 35 insertions(+), 8 deletions(-) diff --git a/README.rst b/README.rst index 515d2ed..27121fc 100644 --- a/README.rst +++ b/README.rst @@ -167,6 +167,8 @@ A location has a ``type`` of either ``publisher`` or ``subscriber``. It supports - ``create_on_post`` (publisher only): if set to ``false``, you will need to create a channel with a PUT request first before POSTing any data to it (default to ``true``) - ``create_on_get`` (subscriber only): if set to ``true``, a non-existing channel will be automatically created at the first GET request (default to ``false``) - ``allow_origin`` (subscriber only): value of ``Access-Control-Allow-Origin`` header send as defined by Cross-Origin Resource Sharing specification (default to ``*``) +- ``allow_credentials`` (subscriber only): value of ``Access-Control-Allow-Credentials`` header send as defined by Cross-Origin Resource Sharing specification (default to ``False``); cannot be ``True`` if ``allow_origin`` is set to ``*`` +- ``passthrough`` (subscriber only): if set to an URL, client's request headers will be passthrough to the given URL every time client subscribes or unsubscribes (default to ``None``) For info, the default configuration looks like this:: diff --git a/bin/hbpushd b/bin/hbpushd index 18f3b44..6946843 100755 --- a/bin/hbpushd +++ b/bin/hbpushd @@ -21,6 +21,8 @@ default_location = { 'create_on_get': False, 'store': 'default', 'allow_origin': '*', + 'allow_credentials': False, + 'passthrough': None, }, 'publisher': { 'create_on_post': True, diff --git a/hbpush/channel.py b/hbpush/channel.py index d8c6ab5..a713345 100644 --- a/hbpush/channel.py +++ b/hbpush/channel.py @@ -1,6 +1,9 @@ +from tornado import httpclient from hbpush.message import Message + import logging import time +import urllib class Channel(object): @@ -23,6 +26,8 @@ def __init__(self, id, store): # Empty message, we just want to keep etag and lastmodified data self.last_message = Message(0, -1) + self.client = httpclient.AsyncHTTPClient() + def send_to_subscribers(self, message): # We work on a copy to deal with reentering subscribers subs = self.subscribers.copy() @@ -46,21 +51,34 @@ def _process_message(message): message = self.make_message(content_type, body) self.store.post(self.id, message, callback=_process_message, errback=errback) - def wait_for(self, last_modified, etag, id_subscriber, callback, errback): + def wait_for(self, last_modified, etag, request, passthrough, id_subscriber, callback, errback): request_msg = Message(last_modified, etag) def _cb(message): if request_msg >= message: - self.subscribe(id_subscriber, _cb, errback) + self.subscribe(id_subscriber, request, passthrough, _cb, errback) else: callback(message) - self.subscribe(id_subscriber, _cb, errback) + self.subscribe(id_subscriber, request, passthrough, _cb, errback) + + def _passthrough(self, action, request, passthrough): + if not passthrough or request.method != 'GET': + return + + def ignore(response): + pass + + url = passthrough + body = urllib.urlencode({'channel_id': self.id, action: 1}) + self.client.fetch(url, ignore, method='POST', body=body, headers=request.headers) - def subscribe(self, id_subscriber, callback, errback): + def subscribe(self, id_subscriber, request, passthrough, callback, errback): + self._passthrough('subscribe', request, passthrough) self.subscribers[id_subscriber] = (callback, errback) - def unsubscribe(self, id_subscriber): + def unsubscribe(self, id_subscriber, request, passthrough): + self._passthrough('unsubscribe', request, passthrough) self.subscribers.pop(id_subscriber, None) def get(self, last_modified, etag, callback, errback): diff --git a/hbpush/pubsub/__init__.py b/hbpush/pubsub/__init__.py index 3b57d17..25d40cd 100644 --- a/hbpush/pubsub/__init__.py +++ b/hbpush/pubsub/__init__.py @@ -15,6 +15,9 @@ class PubSubHandler(RequestHandler): def __init__(self, *args, **kwargs): self.registry = kwargs.pop('registry', None) self.allow_origin = kwargs.pop('allow_origin', '*') + self.allow_credentials = kwargs.pop('allow_credentials', False) + if (self.allow_origin == '*' and self.allow_credentials): + raise AttributeError("allow_origin cannot be '*' with allow_credentials set to true") super(PubSubHandler, self).__init__(*args, **kwargs) def add_vary_header(self): @@ -22,8 +25,9 @@ def add_vary_header(self): def add_accesscontrol_headers(self): self.set_header('Access-Control-Allow-Origin', self.allow_origin) - self.set_header('Access-Control-Allow-Headers', 'If-Modified-Since, If-None-Match') + self.set_header('Access-Control-Allow-Headers', 'If-Modified-Since, If-None-Match, X-Cookie') self.set_header('Access-Control-Expose-Headers', 'Last-Modified, Etag, Cache-Control') + self.set_header('Access-Control-Allow-Credentials', 'true' if self.allow_credentials else 'false') self.set_header('Access-Control-Max-Age', '864000') def _handle_request_exception(self, e): diff --git a/hbpush/pubsub/subscriber.py b/hbpush/pubsub/subscriber.py index 75a854d..3942698 100644 --- a/hbpush/pubsub/subscriber.py +++ b/hbpush/pubsub/subscriber.py @@ -8,6 +8,7 @@ class Subscriber(PubSubHandler): def __init__(self, *args, **kwargs): self.create_on_get = kwargs.pop('create_on_get', False) + self.passthrough = kwargs.pop('passthrough', None) super(Subscriber, self).__init__(*args, **kwargs) @asynchronous @@ -47,7 +48,7 @@ def _process_channel(self, last_modified, etag, channel): class LongPollingSubscriber(Subscriber): def unsubscribe(self): if hasattr(self, 'channel'): - self.channel.unsubscribe(id(self)) + self.channel.unsubscribe(id(self), self.request, self.passthrough) on_connection_close = unsubscribe def finish(self, chunk=None): @@ -58,7 +59,7 @@ def _process_channel(self, last_modified, etag, channel): @self.async_callback def _wait_for_message(error): if error.__class__ == Channel.NotModified: - self.channel.wait_for(last_modified, etag, id(self), callback=self.async_callback(self._process_message), errback=self.errback) + self.channel.wait_for(last_modified, etag, self.request, self.passthrough, id(self), callback=self.async_callback(self._process_message), errback=self.errback) else: self.errback(error) From e1d88bb33365e13529d5817c4625cc88517d10cd Mon Sep 17 00:00:00 2001 From: Mitar Date: Mon, 9 Apr 2012 05:25:53 +0200 Subject: [PATCH 08/14] Version bump. --- hbpush/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hbpush/__init__.py b/hbpush/__init__.py index 3a83701..9ce3d6c 100644 --- a/hbpush/__init__.py +++ b/hbpush/__init__.py @@ -1,2 +1,2 @@ -VERSION = (0, 1, 0) +VERSION = (0, 1, 2) __version__ = '.'.join(map(str, VERSION)) From 4a9dbdf8a81e0b8be62cf5a73bcfcb14188998e8 Mon Sep 17 00:00:00 2001 From: Mitar Date: Mon, 23 Apr 2012 23:10:43 +0200 Subject: [PATCH 09/14] Fix timestamp parsing on Windows. --- hbpush/__init__.py | 2 +- hbpush/pubsub/subscriber.py | 17 +++++++++++++++-- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/hbpush/__init__.py b/hbpush/__init__.py index 9ce3d6c..0b4675a 100644 --- a/hbpush/__init__.py +++ b/hbpush/__init__.py @@ -1,2 +1,2 @@ -VERSION = (0, 1, 2) +VERSION = (0, 1, 3) __version__ = '.'.join(map(str, VERSION)) diff --git a/hbpush/pubsub/subscriber.py b/hbpush/pubsub/subscriber.py index 3942698..9d2319c 100644 --- a/hbpush/pubsub/subscriber.py +++ b/hbpush/pubsub/subscriber.py @@ -4,6 +4,18 @@ from email.utils import formatdate, parsedate_tz, mktime_tz from functools import partial +import logging +import calendar + +# mktime_tz has some problems on Windows (http://bugs.python.org/issue14653), +# so we are converting manually +def convert_timestamp(timestamp): + t = parsedate_tz(timestamp) + if t[9] is None: + return mktime_tz(t) + else: + g = calendar.timegm(t[:9]) + return g - t[9] class Subscriber(PubSubHandler): def __init__(self, *args, **kwargs): @@ -15,8 +27,9 @@ def __init__(self, *args, **kwargs): def get(self, channel_id): try: etag = int(self.request.headers.get('If-None-Match', -1)) - last_modified = int('If-Modified-Since' in self.request.headers and mktime_tz(parsedate_tz(self.request.headers['If-Modified-Since'])) or 0) - except: + last_modified = int('If-Modified-Since' in self.request.headers and convert_timestamp(self.request.headers['If-Modified-Since']) or 0) + except Exception, e: + logging.warning('Error parsing request headers: %s', e) raise HTTPError(400) getattr(self.registry, 'get_or_create' if self.create_on_get else 'get')(channel_id, From 806c04cfc81965bfe237dc3db492e95b1367de42 Mon Sep 17 00:00:00 2001 From: Mitar Date: Sat, 27 Oct 2012 11:21:02 -0700 Subject: [PATCH 10/14] Allow setting server header field used in responses. --- README.rst | 1 + bin/hbpushd | 10 +++++++--- hbpush/pubsub/__init__.py | 6 +++++- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index 27121fc..d63ecb6 100644 --- a/README.rst +++ b/README.rst @@ -88,6 +88,7 @@ You can use the following options: - ``port``: the numeric port number to use (default to ``80``) - ``address``: the IP-address to bind to (default to ``''``) +- ``servername``: server header field used in responses (default to ``None``) Example configuration (in YAML):: diff --git a/bin/hbpushd b/bin/hbpushd index 6946843..863f29f 100755 --- a/bin/hbpushd +++ b/bin/hbpushd @@ -33,6 +33,7 @@ default_location = { defaults = { 'port': 80, 'address': '', + 'servername': None, 'store': { 'type': 'memory', }, @@ -107,7 +108,7 @@ def make_stores(stores_dict): from hbpush.pubsub.publisher import Publisher from hbpush.pubsub.subscriber import Subscriber, LongPollingSubscriber -def make_location(loc_dict, stores={}): +def make_location(loc_dict, stores={}, servername=None): loc_conf = default_location.get(loc_dict['type'], {}).copy() loc_conf.update(loc_dict) @@ -127,14 +128,17 @@ def make_location(loc_dict, stores={}): url = loc_conf.pop('url', loc_conf.pop('prefix', '')+'(.+)') store_id = loc_conf.pop('store') - kwargs = {'registry': stores[store_id]['registry']} + kwargs = { + 'registry': stores[store_id]['registry'], + 'servername': servername, + } kwargs.update(loc_conf) return (url, cls, kwargs) from functools import partial conf['store'] = make_stores(conf['store']) -conf['locations'] = map(partial(make_location, stores=conf['store']), conf['locations']) +conf['locations'] = map(partial(make_location, stores=conf['store'], servername=conf['servername']), conf['locations']) from tornado.web import Application from tornado.httpserver import HTTPServer diff --git a/hbpush/pubsub/__init__.py b/hbpush/pubsub/__init__.py index 25d40cd..a2cce38 100644 --- a/hbpush/pubsub/__init__.py +++ b/hbpush/pubsub/__init__.py @@ -14,6 +14,7 @@ class PubSubHandler(RequestHandler): def __init__(self, *args, **kwargs): self.registry = kwargs.pop('registry', None) + self.servername = kwargs.pop('servername', None) self.allow_origin = kwargs.pop('allow_origin', '*') self.allow_credentials = kwargs.pop('allow_credentials', False) if (self.allow_origin == '*' and self.allow_credentials): @@ -30,6 +31,10 @@ def add_accesscontrol_headers(self): self.set_header('Access-Control-Allow-Credentials', 'true' if self.allow_credentials else 'false') self.set_header('Access-Control-Max-Age', '864000') + def set_default_headers(self): + if self.servername: + self.set_header('Server', self.servername) + def _handle_request_exception(self, e): if e.__class__ in self.exception_mapping: e = HTTPError(self.exception_mapping[e.__class__], str(e)) @@ -38,7 +43,6 @@ def _handle_request_exception(self, e): errback = _handle_request_exception - def simple_finish(self, *args, **kwargs): # ignore everything, and just finish the request self.finish() From 9924ab7dcecbd9d7edcdf91316af0792eab6c28c Mon Sep 17 00:00:00 2001 From: Mitar Date: Sat, 27 Oct 2012 11:21:02 -0700 Subject: [PATCH 11/14] Allow setting server header field used in responses. --- README.rst | 1 + bin/hbpushd | 10 +++++++--- hbpush/pubsub/__init__.py | 6 +++++- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index 27121fc..d63ecb6 100644 --- a/README.rst +++ b/README.rst @@ -88,6 +88,7 @@ You can use the following options: - ``port``: the numeric port number to use (default to ``80``) - ``address``: the IP-address to bind to (default to ``''``) +- ``servername``: server header field used in responses (default to ``None``) Example configuration (in YAML):: diff --git a/bin/hbpushd b/bin/hbpushd index 6946843..863f29f 100755 --- a/bin/hbpushd +++ b/bin/hbpushd @@ -33,6 +33,7 @@ default_location = { defaults = { 'port': 80, 'address': '', + 'servername': None, 'store': { 'type': 'memory', }, @@ -107,7 +108,7 @@ def make_stores(stores_dict): from hbpush.pubsub.publisher import Publisher from hbpush.pubsub.subscriber import Subscriber, LongPollingSubscriber -def make_location(loc_dict, stores={}): +def make_location(loc_dict, stores={}, servername=None): loc_conf = default_location.get(loc_dict['type'], {}).copy() loc_conf.update(loc_dict) @@ -127,14 +128,17 @@ def make_location(loc_dict, stores={}): url = loc_conf.pop('url', loc_conf.pop('prefix', '')+'(.+)') store_id = loc_conf.pop('store') - kwargs = {'registry': stores[store_id]['registry']} + kwargs = { + 'registry': stores[store_id]['registry'], + 'servername': servername, + } kwargs.update(loc_conf) return (url, cls, kwargs) from functools import partial conf['store'] = make_stores(conf['store']) -conf['locations'] = map(partial(make_location, stores=conf['store']), conf['locations']) +conf['locations'] = map(partial(make_location, stores=conf['store'], servername=conf['servername']), conf['locations']) from tornado.web import Application from tornado.httpserver import HTTPServer diff --git a/hbpush/pubsub/__init__.py b/hbpush/pubsub/__init__.py index 25d40cd..a2cce38 100644 --- a/hbpush/pubsub/__init__.py +++ b/hbpush/pubsub/__init__.py @@ -14,6 +14,7 @@ class PubSubHandler(RequestHandler): def __init__(self, *args, **kwargs): self.registry = kwargs.pop('registry', None) + self.servername = kwargs.pop('servername', None) self.allow_origin = kwargs.pop('allow_origin', '*') self.allow_credentials = kwargs.pop('allow_credentials', False) if (self.allow_origin == '*' and self.allow_credentials): @@ -30,6 +31,10 @@ def add_accesscontrol_headers(self): self.set_header('Access-Control-Allow-Credentials', 'true' if self.allow_credentials else 'false') self.set_header('Access-Control-Max-Age', '864000') + def set_default_headers(self): + if self.servername: + self.set_header('Server', self.servername) + def _handle_request_exception(self, e): if e.__class__ in self.exception_mapping: e = HTTPError(self.exception_mapping[e.__class__], str(e)) @@ -38,7 +43,6 @@ def _handle_request_exception(self, e): errback = _handle_request_exception - def simple_finish(self, *args, **kwargs): # ignore everything, and just finish the request self.finish() From deecb6005d2f04181a10024968797c1cd29ddc03 Mon Sep 17 00:00:00 2001 From: Mitar Date: Sat, 27 Oct 2012 11:24:40 -0700 Subject: [PATCH 12/14] Version bump. --- hbpush/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hbpush/__init__.py b/hbpush/__init__.py index 0b4675a..842ec28 100644 --- a/hbpush/__init__.py +++ b/hbpush/__init__.py @@ -1,2 +1,2 @@ -VERSION = (0, 1, 3) +VERSION = (0, 1, 4) __version__ = '.'.join(map(str, VERSION)) From 149f4fdd4e5989acb8ecaf804bc39e95f48a194a Mon Sep 17 00:00:00 2001 From: Mitar Date: Sat, 27 Oct 2012 12:05:04 -0700 Subject: [PATCH 13/14] Bugfix. --- bin/hbpushd | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bin/hbpushd b/bin/hbpushd index 863f29f..3acc48c 100755 --- a/bin/hbpushd +++ b/bin/hbpushd @@ -108,7 +108,10 @@ def make_stores(stores_dict): from hbpush.pubsub.publisher import Publisher from hbpush.pubsub.subscriber import Subscriber, LongPollingSubscriber -def make_location(loc_dict, stores={}, servername=None): +def make_location(loc_dict, stores=None, servername=None): + if stores is None: + stores = {} + loc_conf = default_location.get(loc_dict['type'], {}).copy() loc_conf.update(loc_dict) From 25eec5cba46cd1756649d96bd32de60cdbe07712 Mon Sep 17 00:00:00 2001 From: Mitar Date: Sat, 27 Oct 2012 21:13:23 -0700 Subject: [PATCH 14/14] Cleaned README. --- README.rst | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/README.rst b/README.rst index d63ecb6..b7e345f 100644 --- a/README.rst +++ b/README.rst @@ -242,21 +242,13 @@ Caveats Running Tests ------------- -Make sure you have a test redis server accessible at ``localhost:6379``. **Be careful**, the tests suite will -flush your server default database, you've been warned. +Make sure you have a test redis server accessible at ``localhost:6379``. **Be careful, the tests suite will +flush your server default database, you've been warned.** Run the test suite with :: $ python setup.py nosetests -Known Issues ------------- - -- hbpushd depends on the development version of facebook's tornado. ``setup.py`` will install a - compatible version, but if you have already installed tornado through ``easy_install`` or ``pip``, - you might have some problems with Etags, or when launching hbpushd. In that case, reinstall - the latest version of tornado_. - Change log ----------