/
dim81
/
learning-spark
Обзор
Документация
Войти
/
dim81
/
learning-spark
Код
Запросы
0
Задачи
Вики
Пакеты
0
Релизы
0
Аналитика
Безопасность
master
src/python/BasicMapPartitions.py
35 строк
941 B
Holden Karau
Auto pep8ify the python code
06 сен 2014, 00:13
06 сен 2014, 00:13
2b7d8bd
Код
Авторство
О чём код?
""" >>> from pyspark.context import SparkContext >>> sc = SparkContext('local', 'test') >>> b = sc.parallelize(["KK6JKQ", "Ve3UoW", "kk6jlk", "W6BB"]) >>> fetchCallSigns(b).size() 4 """ import sys import urllib3 from pyspark import SparkContext def processCallSigns(signs): """Process call signs""" http = urllib3.PoolManager() requests = map( lambda x: http.request('GET', "http://qrzcq.com/call/" + x), signs) return map(lambda x: x.data, requests) def fetchCallSigns(input): """Fetch call signs""" return input.mapPartitions(lambda callSigns: processCallSigns(callSigns)) if __name__ == "__main__": master = "local" if len(sys.argv) == 2: master = sys.argv[1] sc = SparkContext(master, "BasicMapPartitions") input = sc.parallelize(["KK6JKQ", "Ve3UoW", "kk6jlk", "W6BB"]) output = sorted(fetchCallSigns(input).collect()) for str in output: print "%s " % (str)