/
githubmirror
/
the-algorithm
Обзор
Документация
Войти
/
githubmirror
/
the-algorithm
Код
Запросы
0
Пакеты
0
Релизы
0
Аналитика
Безопасность
main
src/java/com/twitter/search/earlybird/partition/HdfsUtil.java
30 строк
1016 B
twitter-team
Twitter Recommendation Algorithm
01 апр 2023, 01:36
01 апр 2023, 01:36
ef4c5eb
Код
Авторство
О чём код?
package com.twitter.search.earlybird.partition; import java.io.IOException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; public final class HdfsUtil { private HdfsUtil() { } public static FileSystem getHdfsFileSystem() throws IOException { Configuration config = new Configuration(); // Since earlybird uses hdfs from different threads, and closes the FileSystem from // them independently, we want each thread to have its own, new FileSystem. return FileSystem.newInstance(config); } /** * Checks if the given segment is present on HDFS */ public static boolean segmentExistsOnHdfs(FileSystem fs, SegmentInfo segmentInfo) throws IOException { String hdfsBaseDirPrefix = segmentInfo.getSyncInfo().getHdfsUploadDirPrefix(); FileStatus[] statuses = fs.globStatus(new Path(hdfsBaseDirPrefix)); return statuses != null && statuses.length > 0; } }