迭代器與生成器

手動遍歷迭代器

• 不使用for循環(huán)手動遍歷迭代器

def manual_iter():
    '''
    手動遍歷迭代器
    '''
    with open('/etc/passwd') as f:
        try:
            while True:
                line = next(f)
                print (line)
        #通過StopIteration異常來標記結(jié)尾
        except StopIteration:
            pass

def manual_iter_1():
    with open('/etc/passwd') as f:
        while True:
            line = next(f, None)
            if line is None:
                break
            print (line)

代理迭代

• 在一個新的容器(class)中迭代對象

class Node:
    def __init__(self, value):
        self._value = value
        #存儲Node對象
        self._children = []

    def __repr__(self):
        return 'Node({!r})'.format(self._value)

    def add_child(self, node):
        self._children.append(node)

    def __iter__(self):
        '''
        傳遞迭代請求給內(nèi)部的_children,實現(xiàn)了一個__next()__方法的迭代器對象
        '''
        return iter(self._children)

if __name__ == '__main__':
    root = Node(0)
    child1 = Node(1)
    child2 = Node(2)
    root.add_child(child1)
    root.add_child(child2)
    for ch in root:
        print (ch)

使用生成器創(chuàng)建新的迭代模式

• 生成器只能用于迭代操做，使用for循環(huán)迭代生成器會自動處理StopIteration異常

def frange(start, stop, increment):
    x = start
    while x < stop:
        yield x
        x += increment

def countdown(n):
    print ('Starting to count from', n)
    while n > 0:
        yield n
        n -= 1
    print ('Done!')

實現(xiàn)迭代器協(xié)議

• 構(gòu)建一個支持迭代操作的自定義對象

class Node:
    def __init__(self, value):
        self._value = value
        #存儲Node對象
        self._children = []

    def __repr__(self):
        return 'Node({!r})'.format(self._value)

    def add_child(self, node):
        self._children.append(node)

    def __iter__(self):
        #傳遞迭代請求給內(nèi)部的_children
        return iter(self._children)
        #return self._children.__iter__()

    def depth_first(self):
        #返回自己本身的一個迭代
        yield self
        for c in self:
            #調(diào)用迭代本身的函數(shù)
            yield from c.depth_first()

if __name__ == '__main__':
    #根
    root = Node(0)
    child1 = Node(1)
    child2 = Node(2)
    root.add_child(child1)
    root.add_child(child2)
    child1.add_child(Node(3))
    child1.add_child(Node(4))
    child2.add_child(Node(5))
    for ch in root.depth_first():
        print (ch)

反向迭代

• 使用內(nèi)置的reversed()
• 反向迭代僅僅當對象的大小可預先確定或者對象實現(xiàn)了 reversed() 的特殊方法時才能生效

>>> a = [1, 2, 3, 4]
>>> for x in reversed(a):
...     print (x)
4
3
2
1

• 通過reversed自定義類的反向迭代

class Countdown:

    def __init__(self, start):
        self.start = start

    def __iter__(self):
        n = self.start
        while n > 0:
            yield n
            n -= 1

    def __reversed__(self):
        n = 1
        while n <= self.start:
            yield n
            n += 1

if __name__ == '__main__':
    #調(diào)用自己實現(xiàn)的__reversed__方法
    for rr in reversed(Countdown(30)):
        print (rr)
    for rr in Countdown(30):
        print (rr)

帶有外部狀態(tài)的生成器函數(shù)

• 定義一個生成器函數(shù)，調(diào)用某個你想暴露給用戶使用的外部狀態(tài)值
• 通過類的iter函數(shù)實現(xiàn)

class LineHistory:
    def __init__(self, lines, histlen=3):
        self.lines = lines
        self.history = deque(maxlen=histlen)

    def __iter__(self):
        #遍歷并標記計數(shù)起始點
        for lineno, line in enumerate(self.lines, 1):
            self.history.append((lineno, line))
            yield line

    def clear(self):
        self.history.clear()

if __name__ == '__main__':
    with open('ch4_1.py') as f:
        lines = LineHistory(f)
        #__iter_函數(shù)返回的生成器 
        for line in lines:
            if 'python' in line:
                for lineno, hline in lines.history:
                    print ('{}:{}'.format(lineno, hline))

迭代器切片

• 得到一個由迭代器生成的切片對象
• 迭代器和生成器不能使用標準的切片操作，因為它們的長度事先我們并不知道(并且也沒有實現(xiàn)索引)。 函數(shù) islice() 返回一個可以生成指定元素的迭代器，它通過遍歷并丟棄直到切片開始索引位置的所有元素。 然后才開始一個個的返回元素，并直到切片結(jié)束索引位置。
• islice() 會消耗掉傳入的迭代器中的數(shù)據(jù)

import itertools
def count(n):
    while True:
        yield n
        n += 1

if __name__ == '__main__':
    c = count(0)
    for x in itertools.islice(c, 10, 20):
        print (x)

跳過可迭代對象的開始部分

• 跳過具體部分(開頭是#的文件)

from itertools import dropwhile
def open_passwd():
    with open('/etc/passwd') as f:
        for line in dropwhile(lambda line:line.startswith('#'), f):
            print (line)

• 如果明確知道跳過的個數(shù)

>>> from itertools import islice
>>> items = ['a', 'b', 'c', 1, 2, 3]
>>> for x in islice(items, 3, None):
...     print (x)
1
2
3

排列組合迭代

• 迭代遍歷一個集合中元素的所有可能的排列或組合
• permutations接受一個集合并產(chǎn)生一個元組序列，每個元組由集合中所有元素的一個可能排列組成

>>> items = ['a', 'b', 'c']
>>> from itertools import permutations
>>> for p in permutations(items):
...     print (p)
('a', 'b', 'c')
('a', 'c', 'b')
('b', 'a', 'c')
('b', 'c', 'a')
('c', 'a', 'b')
('c', 'b', 'a')

• 指定長度的排列

>>> for p in permutations(items, 2):
...     print (p)
('a', 'b')
('a', 'c')
('b', 'a')
('b', 'c')
('c', 'a')
('c', 'b')

• 指定長度的組合(不考慮排序)

>>> from itertools import combinations
>>> for c in combinations(items, 3):
...     print (c)
('a', 'b', 'c')
>>> for c in combinations(items, 2):
...     print (c)
('a', 'b')
('a', 'c')
('b', 'c')

序列上索引值迭代

• enumerate()可以很好解決
• 指定一個start，改變索引開始的值

>>> for id, val in enumerate(my_list, 1):
...     print (id, val)
...     
... 
1 a
2 b
3 c

• 在遍歷文件時定位錯誤行數(shù)

def pares_data(filename):
    with open(filename, 'r') as f:
        for lineno, line in enumerate(f, 1):
            fields = line.split()
            try:
                count = int(fields[0])
            except ValueError as e:
                print ('Line {}: Parse error:{}'.format(lineno, e))

• 跟蹤某些單詞在出現(xiàn)的函數(shù)，構(gòu)造出一個字典

def line_dict():
    from collections import defaultdict
    #初始化字典,value=list
    word_summary = defaultdict(list)
    with open('ch4_1.py', 'r') as f:
        lines = f.readlines()

    for idx, line in enumerate(lines):
        words = [w.strip().lower() for w in line.split()]
        for word in words:
            word_summary[word].append(idx)
    return word_summary

同時迭代多個序列

• 使用zip(), 迭代的次數(shù)和最短的序列一致

>>> xpts = [1, 5, 4, 2, 10, 7]
>>> ypts = [101, 78, 37, 15, 52, 99]
>>> for x, y in zip(xpts, ypts):
...     print (x, y)
1 101
5 78
4 37
2 15
10 52
7 99

• 可以使用itertools.zip_longest()來填充沒有zip迭代到的默認值

>>> for i in zip_longest(a, b):
...     print (i)
(1, 'w')
(2, 'x')
(3, 'y')
(None, 'z')
>>> for i in zip_longest(a, b, fillvalue=0):
...     print (i)
(1, 'w')
(2, 'x')
(3, 'y')
(0, 'z')

• 如果zip的列表一個是key，一個是value,可以打包成字典

>>> headers = ['name', 'shares', 'price']
>>> values = ['ACME', 100, 490.1]
>>> s = dict(zip(headers, values))
>>> print (s) {'name': 'ACME', 'shares': 100, 'price': 490.1}

不同集合上元素的迭代

• 在不同的可迭代對象上循環(huán)變量(避免寫重復的for循環(huán))

>>> from itertools import chain
>>> a = [1, 2, 3, 4]
>>> b = ['x', 'y', 'z']
>>> for x in chain(a, b):
...     print (x)
1
2
3
4
x
y
z

數(shù)據(jù)處理管道

import os
import fnmatch
import gzip
import bz2
import re

def gen_find(filepat, top):
    #遍歷當期文件夾的所有內(nèi)容,返回當期文件夾路徑,文件夾列表，文件列表
    for path, dirlist, filelist in os.walk(top):
        #找到匹配的日志,重新組裝路徑
        for name in fnamatch.filter(filelist, filepat):
            yield os.path.join(path, name)

def gen_opener(filenames):
    #每次返回一個打開的文件描述符
    for filename in filenames:
        if filename.endswith('.gz'):
            f = gzip.open(filename, 'tr')
        elif filename.endswith('.bz2'):
            f = bz2.open(filename, 'rt')
        else:
            f = open(filename, 'rt')
        yield f
        f.close()

def gen_concatenate(iterators):
    #拼接打開的內(nèi)容
    for it in iterators:
        yield from it

def gen_grep(pattern, lines):
    pat = re.complate(pattern)
    for line in lines:
        if pat.search(line):
            yield line

if __name__ == '__main__':
    lognames = gen_find('access-log*', 'www')
    files = gen_opener(lognames)
    pylines = gen_grep('(?i)python', lines)
    for line in pylines:
        print (lines)

展開嵌套的序列

• 將一個多層嵌套序列展開為單層列表
• Iterable檢查元素是否是可以迭代的,ignore_types防止迭代字符數(shù)據(jù)

from collections import Iterable
def flatten(items, ignore_types=(str, bytes)):
    for x in items:
        if isinstance(x, Iterable) and not isinstance(x, ignore_types):
            yield from flatten(x)
        else:
            yield x

if __name__ == '__main__':
    items = [1, 2, [3, 4, [5, 6], 7], 8]
    for x in flatten(items):
        print (x)

順序迭代合并后的序列

• 要求輸入序列是經(jīng)過排序的

>>> import heapq
>>> a  = [1, 4, 7, 10]
>>> b = [2, 5, 6, 11]
>>> for c in heapq.merge(a, b):
...     print (c)
1
2
4
5
6
7
10
11