Genrators

Fibonacci Sequence

from functools import lru_cache

@lru_cache()
def fib_recursive(n):
    if n <= 1:
        return 1
    else:
        return fib_recursive(n-1) + fib_recursive(n-2)

let's create an iterator approach so we can iterate over the sequence, but without materializing it

def fib_gen(n):
    fib_0 = 1
    yield fib_0
    fib_1 = 1
    yield fib_1
    for i in range(n-1):
        fib_0, fib_1 = fib_1, fib_0 + fib_1
        yield fib_1    

[num for num in fib_gen(7)]
Out:
[1, 1, 2, 3, 5, 8, 13, 21]

Making an Iterable from a Generator

As we now know, generators are iterators.

This means that they become exhausted - so sometimes we want to create an iterable instead.

There's no magic here, we simply have to implement a class that implements the iterable protocol:

def squares_gen(n):
    for i in range(n):
        yield i ** 2

sq = squares_gen(5)
for num in sq:
    print(num) 
0
1
4
9
16

next(sq)
StopIteration                

But, sq was an iterator - so now it's been exhausted, To restart the iteration we have to create a new instance of the generator,let's wrap this in an iterable:

class Squares:
    def __init__(self, n):
        self.n = n
        
    @staticmethod
    def squares_gen(n):
        for i in range(n):
            yield i ** 2
        
    def __iter__(self):
        return Squares.squares_gen(self.n)

Card Deck

from collections import namedtuple

Card = namedtuple('Card', 'rank, suit')
SUITS = ('Spades', 'Hearts', 'Diamonds', 'Clubs')
RANKS = tuple(range(2, 11)) + tuple('JQKA')

def card_gen():
    for suit in SUITS:
        for rank in RANKS:
            card = Card(rank, suit)
            yield card

for card in card_gen():
    print(card)
Card(rank=2, suit='Spades')
Card(rank=3, suit='Spades')
Card(rank=4, suit='Spades')
Card(rank=5, suit='Spades')
Card(rank=6, suit='Spades')
Card(rank=7, suit='Spades')......

We can now make it into an iterable:

deck = CardDeck()
print(len[card for card in deck])

54

One thing we don't have here is the support for reversed,but we can add it in by implementing the __reversed__ method and returning an iterator that iterates the deck in reverse order.

class CardDeck:
    SUITS = ('Spades', 'Hearts', 'Diamonds', 'Clubs')
    RANKS = tuple(range(2, 11)) + tuple('JQKA')
        
    def __iter__(self):
        return CardDeck.card_gen()
    
    def __reversed__(self):
        return CardDeck.reversed_card_gen()
    
    @staticmethod
    def card_gen():
        for suit in CardDeck.SUITS:
            for rank in CardDeck.RANKS:
                card = Card(rank, suit)
                yield card
        
    @staticmethod
    def reversed_card_gen():
        for suit in reversed(CardDeck.SUITS):
            for rank in reversed(CardDeck.RANKS):
                card = Card(rank, suit)
                yield card


rev = reversed(CardDeck())
[card for card in rev]

Out:
[Card(rank='A', suit='Clubs'),
 Card(rank='K', suit='Clubs'),
 Card(rank='Q', suit='Clubs'),
 Card(rank='J', suit='Clubs'),
 Card(rank=10, suit='Clubs'),
 Card(rank=9, suit='Clubs'),
 Card(rank=8, suit='Clubs'),
 Card(rank=7, suit='Clubs'),
 Card(rank=6, suit='Clubs'),
 Card(rank=5, suit='Clubs'),
 Card(rank=4, suit='Clubs'),
 Card(rank=3, suit='Clubs'),
 Card(rank=2, suit='Clubs'),
 Card(rank='A', suit='Diamonds'),
 Card(rank='K', suit='Diamonds'),
 Card(rank='Q', suit='Diamonds'),
 Card(rank='J', suit='Diamonds'),
 Card(rank=10, suit='Diamonds'),
 Card(rank=9, suit='Diamonds'),
 Card(rank=8, suit='Diamonds'),
 Card(rank=7, suit='Diamonds'),
 Card(rank=6, suit='Diamonds'),
 Card(rank=5, suit='Diamonds'),
 Card(rank=4, suit='Diamonds'),
 Card(rank=3, suit='Diamonds'),
 Card(rank=2, suit='Diamonds'),
 Card(rank='A', suit='Hearts'),
 Card(rank='K', suit='Hearts'),
 Card(rank='Q', suit='Hearts'),
 Card(rank='J', suit='Hearts'),
 Card(rank=10, suit='Hearts'),
 Card(rank=9, suit='Hearts'),
 Card(rank=8, suit='Hearts'),
 Card(rank=7, suit='Hearts'),
 Card(rank=6, suit='Hearts'),
 Card(rank=5, suit='Hearts'),
 Card(rank=4, suit='Hearts'),
 Card(rank=3, suit='Hearts'),
 Card(rank=2, suit='Hearts'),
 Card(rank='A', suit='Spades'),
 Card(rank='K', suit='Spades'),
 Card(rank='Q', suit='Spades'),
 Card(rank='J', suit='Spades'),
 Card(rank=10, suit='Spades'),
 Card(rank=9, suit='Spades'),
 Card(rank=8, suit='Spades'),
 Card(rank=7, suit='Spades'),
 Card(rank=6, suit='Spades'),
 Card(rank=5, suit='Spades'),
 Card(rank=4, suit='Spades'),
 Card(rank=3, suit='Spades'),
 Card(rank=2, suit='Spades')]

Yield From

Example

Here's an example where using yield from can be quite effective.

In this example we need to read car brands from multiple files to get it as a single collection.

We might do it this way:

brands = []

with open('car-brands-1.txt') as f:
    for brand in f:
        brands.append(brand.strip('\n'))
        
with open('car-brands-2.txt') as f:
    for brand in f:
        brands.append(brand.strip('\n'))
        
with open('car-brands-3.txt') as f:
    for brand in f:
        brands.append(brand.strip('\n'))


for brand in brands:
    print(brand, end=', ')

We can simplify our function by using yield from:

def brands(*files):
    for f_name in files:
        with open(f_name) as f:
            yield from f

for brand in brands(*files):
    print(brand, end=', ')

Alfa Romeo
, Aston Martin
, Audi
, Bentley
, Benz
, BMW
, Bugatti
, Cadillac
, Chevrolet
, Chrysler
, Citroën
, Corvette
, DAF
, Dacia
, Daewoo
, Daihatsu
......

So, we are going to create generators that can read each line of the file, and yield a clean result, and we'll yield from that generator:

def gen_clean_read(file):
    with open(file) as f:
        for line in f:
            yield line.strip('\n')

As you can see, this generator function will clean each line of the file before yielding it. Let's try it with a single file and make sure it works:

f1 = gen_clean_read('car-brands-1.txt')
for line in f1:
    print(line, end=', ')

Alfa Romeo, Aston Martin, Audi, Bentley, Benz, BMW,Bugatti...

Ok, that works. So now, we can proceed with our overarching generator function as before, except we'll yield from our generators, instead of directly from the file iterator:

files = 'car-brands-1.txt', 'car-brands-2.txt', 'car-brands-3.txt'

def brands(*files):
    for file in files:
        yield from gen_clean_read(file)

for brand in brands(*files):
    print(brand, end=', ')

Alfa Romeo, Aston Martin, Audi, Bentley, Benz, BMW,Bugatti...

Other examples

# 注：为了加强示例代码的说明性，本文中的部分代码片段使用了Python 3.5
# 版本添加的 Type Hinting 特性

def add_ellipsis(comments: typing.List[str], max_length: int = 12):
    """如果评论列表里的内容超过 max_length，剩下的字符用省略号代替
    """
    index = 0
    for comment in comments:
        comment = comment.strip()
        if len(comment) > max_length:
            comments[index] = comment[:max_length] + '...'
        index += 1
    return comments


comments = [
    "Implementation note",
    "Changed",
    "ABC for generator",
]
print("\n".join(add_ellipsis(comments)))

Implementati...
Changed
ABC for gene...

============================================================

def add_ellipsis_gen(comments: typing.Iterable[str], max_length: int = 12):
    """如果可迭代评论里的内容超过 max_length，剩下的字符用省略号代替
    """
    for comment in comments:
        comment = comment.strip()
        if len(comment) > max_length:
            yield comment[:max_length] + '...'
        else:
            yield comment


print("\n".join(add_ellipsis_gen(comments)))

Implementati...
Changed
ABC for gene...