A Generator That Paginates

Step 5: A Generator That Paginates

Writing iter and next by hand is explicit but verbose. Python's generators give you the same lazy iterator for free: any function containing yield returns a generator, and its body runs only as values are pulled.

Write paginate(source) as a generator that yields every item across all pages of a PagedSource, fetching each page only when the previous one has been fully yielded. Because yield pauses execution between items, a consumer that stops early, say via itertools.islice, never forces the later pages to load. Same Iterator pattern, a fraction of the code.

Hints
Starter code
class ListCursor:
    def __init__(self, items):
        self._items = list(items)
        self._index = 0

    def __iter__(self):
        return self

    def __next__(self):
        if self._index >= len(self._items):
            raise StopIteration
        value = self._items[self._index]
        self._index += 1
        return value

    def take(self, n):
        result = []
        for _ in range(n):
            try:
                result.append(next(self))
            except StopIteration:
                break
        return result


class PagedSource:
    def __init__(self, items, page_size):
        self.items = list(items)
        self.page_size = page_size
        self.fetches = 0

    def page(self, index):
        self.fetches += 1
        start = index * self.page_size
        return self.items[start:start + self.page_size]

class Cursor:
    def __init__(self, source):
        self.source = source
        self._page_index = 0
        self._buffer = []
        self._pos = 0
        self._done = False

    def __iter__(self):
        return self

    def __next__(self):
        while self._pos >= len(self._buffer):
            if self._done:
                raise StopIteration
            page = self.source.page(self._page_index)
            self._page_index += 1
            if not page:
                self._done = True
                raise StopIteration
            self._buffer = page
            self._pos = 0
        value = self._buffer[self._pos]
        self._pos += 1
        return value

    def take(self, n):
        result = []
        for _ in range(n):
            try:
                result.append(next(self))
            except StopIteration:
                break
        return result


def paginate(source):
    """Yield every item, one page at a time, fetching pages lazily as you go."""
    # TODO: turn this into a generator (use yield):
    #   index = 0
    #   while True:
    #       page = source.page(index)
    #       if not page:
    #           return
    #       for item in page:
    #           yield item
    #       index += 1
    pass
Write it on a laptop

The editor and the test runner need a wide screen. Open this page on a desktop browser and your progress will be waiting.