迭代器與生成器
手動遍歷迭代器
def manual_iter():
'''
手動遍歷迭代器
'''
with open('/etc/passwd') as f:
try:
while True:
line = next(f)
print (line)
#通過StopIteration異常來標記結尾
except StopIteration:
pass
def manual_iter_1():
with open('/etc/passwd') as f:
while True:
line = next(f, None)
if line is None:
break
print (line)
代理迭代
class Node:
def __init__(self, value):
self._value = value
#存儲Node對象
self._children = []
def __repr__(self):
return 'Node({!r})'.format(self._value)
def add_child(self, node):
self._children.append(node)
def __iter__(self):
'''
傳遞迭代請求給內部的_children,實現了一個__next()__方法的迭代器對象
'''
return iter(self._children)
if __name__ == '__main__':
root = Node(0)
child1 = Node(1)
child2 = Node(2)
root.add_child(child1)
root.add_child(child2)
for ch in root:
print (ch)
使用生成器創建新的迭代模式
- 生成器只能用于迭代操做，使用for循環迭代生成器會自動處理StopIteration異常
def frange(start, stop, increment):
x = start
while x < stop:
yield x
x += increment
def countdown(n):
print ('Starting to count from', n)
while n > 0:
yield n
n -= 1
print ('Done!')
實現迭代器協議
class Node:
def __init__(self, value):
self._value = value
#存儲Node對象
self._children = []
def __repr__(self):
return 'Node({!r})'.format(self._value)
def add_child(self, node):
self._children.append(node)
def __iter__(self):
#傳遞迭代請求給內部的_children
return iter(self._children)
#return self._children.__iter__()
def depth_first(self):
#返回自己本身的一個迭代
yield self
for c in self:
#調用迭代本身的函數
yield from c.depth_first()
if __name__ == '__main__':
#根
root = Node(0)
child1 = Node(1)
child2 = Node(2)
root.add_child(child1)
root.add_child(child2)
child1.add_child(Node(3))
child1.add_child(Node(4))
child2.add_child(Node(5))
for ch in root.depth_first():
print (ch)
反向迭代
- 使用內置的reversed()
- 反向迭代僅僅當對象的大小可預先確定或者對象實現了 reversed() 的特殊方法時才能生效
>>> a = [1, 2, 3, 4]
>>> for x in reversed(a):
... print (x)
4
3
2
1
class Countdown:
def __init__(self, start):
self.start = start
def __iter__(self):
n = self.start
while n > 0:
yield n
n -= 1
def __reversed__(self):
n = 1
while n <= self.start:
yield n
n += 1
if __name__ == '__main__':
#調用自己實現的__reversed__方法
for rr in reversed(Countdown(30)):
print (rr)
for rr in Countdown(30):
print (rr)
帶有外部狀態的生成器函數
- 定義一個生成器函數，調用某個你想暴露給用戶使用的外部狀態值
- 通過類的iter函數實現
class LineHistory:
def __init__(self, lines, histlen=3):
self.lines = lines
self.history = deque(maxlen=histlen)
def __iter__(self):
#遍歷并標記計數起始點
for lineno, line in enumerate(self.lines, 1):
self.history.append((lineno, line))
yield line
def clear(self):
self.history.clear()
if __name__ == '__main__':
with open('ch4_1.py') as f:
lines = LineHistory(f)
#__iter_函數返回的生成器
for line in lines:
if 'python' in line:
for lineno, hline in lines.history:
print ('{}:{}'.format(lineno, hline))
迭代器切片
- 得到一個由迭代器生成的切片對象
- 迭代器和生成器不能使用標準的切片操作，因為它們的長度事先我們并不知道(并且也沒有實現索引)。 函數 islice() 返回一個可以生成指定元素的迭代器，它通過遍歷并丟棄直到切片開始索引位置的所有元素。 然后才開始一個個的返回元素，并直到切片結束索引位置。
- islice() 會消耗掉傳入的迭代器中的數據
import itertools
def count(n):
while True:
yield n
n += 1
if __name__ == '__main__':
c = count(0)
for x in itertools.islice(c, 10, 20):
print (x)
跳過可迭代對象的開始部分
from itertools import dropwhile
def open_passwd():
with open('/etc/passwd') as f:
for line in dropwhile(lambda line:line.startswith('#'), f):
print (line)
>>> from itertools import islice
>>> items = ['a', 'b', 'c', 1, 2, 3]
>>> for x in islice(items, 3, None):
... print (x)
1
2
3
排列組合迭代
- 迭代遍歷一個集合中元素的所有可能的排列或組合
- permutations接受一個集合并產生一個元組序列，每個元組由集合中所有元素的一個可能排列組成
>>> items = ['a', 'b', 'c']
>>> from itertools import permutations
>>> for p in permutations(items):
... print (p)
('a', 'b', 'c')
('a', 'c', 'b')
('b', 'a', 'c')
('b', 'c', 'a')
('c', 'a', 'b')
('c', 'b', 'a')
>>> for p in permutations(items, 2):
... print (p)
('a', 'b')
('a', 'c')
('b', 'a')
('b', 'c')
('c', 'a')
('c', 'b')
>>> from itertools import combinations
>>> for c in combinations(items, 3):
... print (c)
('a', 'b', 'c')
>>> for c in combinations(items, 2):
... print (c)
('a', 'b')
('a', 'c')
('b', 'c')
序列上索引值迭代
- enumerate()可以很好解決
- 指定一個start，改變索引開始的值
>>> for id, val in enumerate(my_list, 1):
... print (id, val)
...
...
1 a
2 b
3 c
def pares_data(filename):
with open(filename, 'r') as f:
for lineno, line in enumerate(f, 1):
fields = line.split()
try:
count = int(fields[0])
except ValueError as e:
print ('Line {}: Parse error:{}'.format(lineno, e))
def line_dict():
from collections import defaultdict
#初始化字典,value=list
word_summary = defaultdict(list)
with open('ch4_1.py', 'r') as f:
lines = f.readlines()
for idx, line in enumerate(lines):
words = [w.strip().lower() for w in line.split()]
for word in words:
word_summary[word].append(idx)
return word_summary
同時迭代多個序列
>>> xpts = [1, 5, 4, 2, 10, 7]
>>> ypts = [101, 78, 37, 15, 52, 99]
>>> for x, y in zip(xpts, ypts):
... print (x, y)
1 101
5 78
4 37
2 15
10 52
7 99
- 可以使用itertools.zip_longest()來填充沒有zip迭代到的默認值
>>> for i in zip_longest(a, b):
... print (i)
(1, 'w')
(2, 'x')
(3, 'y')
(None, 'z')
>>> for i in zip_longest(a, b, fillvalue=0):
... print (i)
(1, 'w')
(2, 'x')
(3, 'y')
(0, 'z')
- 如果zip的列表一個是key，一個是value,可以打包成字典
>>> headers = ['name', 'shares', 'price']
>>> values = ['ACME', 100, 490.1]
>>> s = dict(zip(headers, values))
>>> print (s) {'name': 'ACME', 'shares': 100, 'price': 490.1}
不同集合上元素的迭代
- 在不同的可迭代對象上循環變量(避免寫重復的for循環)
>>> from itertools import chain
>>> a = [1, 2, 3, 4]
>>> b = ['x', 'y', 'z']
>>> for x in chain(a, b):
... print (x)
1
2
3
4
x
y
z
數據處理管道
import os
import fnmatch
import gzip
import bz2
import re
def gen_find(filepat, top):
#遍歷當期文件夾的所有內容,返回當期文件夾路徑,文件夾列表，文件列表
for path, dirlist, filelist in os.walk(top):
#找到匹配的日志,重新組裝路徑
for name in fnamatch.filter(filelist, filepat):
yield os.path.join(path, name)
def gen_opener(filenames):
#每次返回一個打開的文件描述符
for filename in filenames:
if filename.endswith('.gz'):
f = gzip.open(filename, 'tr')
elif filename.endswith('.bz2'):
f = bz2.open(filename, 'rt')
else:
f = open(filename, 'rt')
yield f
f.close()
def gen_concatenate(iterators):
#拼接打開的內容
for it in iterators:
yield from it
def gen_grep(pattern, lines):
pat = re.complate(pattern)
for line in lines:
if pat.search(line):
yield line
if __name__ == '__main__':
lognames = gen_find('access-log*', 'www')
files = gen_opener(lognames)
pylines = gen_grep('(?i)python', lines)
for line in pylines:
print (lines)
展開嵌套的序列
- 將一個多層嵌套序列展開為單層列表
- Iterable檢查元素是否是可以迭代的,ignore_types防止迭代字符數據
from collections import Iterable
def flatten(items, ignore_types=(str, bytes)):
for x in items:
if isinstance(x, Iterable) and not isinstance(x, ignore_types):
yield from flatten(x)
else:
yield x
if __name__ == '__main__':
items = [1, 2, [3, 4, [5, 6], 7], 8]
for x in flatten(items):
print (x)
順序迭代合并后的序列
>>> import heapq
>>> a = [1, 4, 7, 10]
>>> b = [2, 5, 6, 11]
>>> for c in heapq.merge(a, b):
... print (c)
1
2
4
5
6
7
10
11
