Hive之外部分區表

本文介紹了如何在Hive里新建一個外部分區表并加載數據

1.建表

# 使用數據庫
use blog;

# 創建外部分區表
create external table external_blog_record(
    host string comment "主機",
    app string comment "應用",
    source string comment "來源",
    remote_addr string comment "訪問IP",
    time_iso6401 string comment "訪問時間",
    http_host string comment "域名",
    request_method string comment "請求方式",
    request_url string comment "請求地址",
    request_protocol string comment "請求協議",
    request_time string comment "請求耗時",
    status string comment "請求狀態",
    body_byte_sents string comment "內容體大小",
    upstream_addr string comment "轉發服務器地址",
    upstream_response_time string comment "轉發響應耗時",
    upstream_status string comment "轉發狀態",
    http_referer string comment "來源地址",
    http_user_agent string comment "瀏覽器類型",
    res_type string comment "資源類型:首頁、文章、類別、其他"
) 
comment "日志原始記錄外部分區表"
partitioned by (day string) 
row format delimited fields terminated by '\t' 
location '/log/blog';

新建一個名為external_blog_record的數據庫表并制定分區參數day,數據的格式用'\t'分隔,數據的目錄存放在HDFS的'/log/blog'目錄下。

2.查看分區

# 查看表分區
show partitions external_blog_record;
分區列表

可以看到目前表里面已經存在很多分區了,查看HDFS的目錄


hdfs中的分區

每個分區下對應存放這日志文件。


分區中的日志文件

3.新增分區

只需要在 /log/blog 下 新建day=XXX 即可,但是這樣新建的分區并沒有和Hive關聯起來,必須運行如下命令,使分區與Hive關聯起來。

msck repair table external_blog_record;

就可以用上面的查看分區的命令查看是否新建成功。

4.查詢分區下的記錄

hive> select count(*) from external_blog_record where day=20181122;
Query ID = hadoop_20181123144713_2b8b197a-c09b-4bf6-8ad3-b88cbd1ee4ca
Total jobs = 1
Launching Job 1 out of 1
Number of reduce tasks determined at compile time: 1
In order to change the average load for a reducer (in bytes):
  set hive.exec.reducers.bytes.per.reducer=<number>
In order to limit the maximum number of reducers:
  set hive.exec.reducers.max=<number>
In order to set a constant number of reducers:
  set mapreduce.job.reduces=<number>
Starting Job = job_1542348923310_0146, Tracking URL = http://hadoop1:8088/proxy/application_1542348923310_0146/
Kill Command = /opt/soft/hadoop-2.7.3/bin/hadoop job  -kill job_1542348923310_0146
Hadoop job information for Stage-1: number of mappers: 1; number of reducers: 1
2018-11-23 14:47:25,697 Stage-1 map = 0%,  reduce = 0%
2018-11-23 14:47:32,237 Stage-1 map = 100%,  reduce = 0%, Cumulative CPU 1.49 sec
2018-11-23 14:47:39,923 Stage-1 map = 100%,  reduce = 100%, Cumulative CPU 2.62 sec
MapReduce Total cumulative CPU time: 2 seconds 620 msec
Ended Job = job_1542348923310_0146
MapReduce Jobs Launched:
Stage-Stage-1: Map: 1  Reduce: 1   Cumulative CPU: 2.62 sec   HDFS Read: 414396 HDFS Write: 5 SUCCESS
Total MapReduce CPU Time Spent: 2 seconds 620 msec
OK
1561
Time taken: 27.642 seconds, Fetched: 1 row(s)

5.附錄

利用MapperReduce來定時合并小文件并加載到Hive分區表里

/**
 * 合并日志文件并加載到Hive分區表
 */
public class MergeSmallFileAndLoadIntoHive {

    private static final Logger LOG = LoggerFactory.getLogger(MergeSmallFileAndLoadIntoHive.class);

    static class SmallFileCombinerMapper extends Mapper<LongWritable, Text, Text, NullWritable> {
        NullWritable v = NullWritable.get();

        @Override
        protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
            context.write(value, v);
        }
    }

    public static void main(String[] args) throws Exception {

        boolean test = false;
        String logPath;
        String patition;

        if (test) {
            patition = "day=20181114";

            // Linux
            logPath = "/log/blog";

            // Windows
            logPath = "D:" + File.separator + "hadoop" + File.separator + "blog";
        } else {
            if (args == null || args.length < 2) {
                throw new RuntimeException("\"參數的長度不正確,參考:[java -jar xxxx.jar me.jinkun.mr.merge.MergeSmallFileAndLoadIntoHive /log/blog day=20181116]\"");
            }

            logPath = args[0];
            patition = args[1];
        }


        String tempInPath = logPath + File.separator + "temp" + File.separator + patition + File.separator + "in";
        String tempOutPath = logPath + File.separator + "temp" + File.separator + patition + File.separator + "out";

        //權限問題
        System.setProperty("HADOOP_USER_NAME", "hadoop");

        Configuration conf = new Configuration();
        if (!test) {
            conf.set("fs.defaultFS", "hdfs://hadoop1:9000");
        }

        // 1.獲取當天臨時保存的日志
        List<Path> paths = new ArrayList<>();
        long currentTimeMillis = System.currentTimeMillis();
        FileSystem fs = FileSystem.get(conf);
        FileStatus[] fileStatuses = fs.listStatus(new Path(tempInPath));
        for (FileStatus fileStatus : fileStatuses) {
            if (fileStatus.isDirectory()) {
                Path path = fileStatus.getPath();
                String name = fileStatus.getPath().getName();
                if (!name.startsWith("delete") &&
                        name.compareTo(String.valueOf(currentTimeMillis)) < 0) {
                    paths.add(path);
                }
                LOG.info("文件夾名為:" + name);
            }
        }

        if (paths.size() == 0) {
            LOG.info("暫無可以合并的文件夾!不提交JOB");
            System.exit(0);
        }

        Job job = Job.getInstance(conf);
        job.setJarByClass(MergeSmallFileAndLoadIntoHive.class);
        job.setMapperClass(SmallFileCombinerMapper.class);
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(NullWritable.class);

        // 2.合并小文件到臨時文件夾
        job.setInputFormatClass(CombineTextInputFormat.class);
        CombineTextInputFormat.setMaxInputSplitSize(job, 1024 * 1024 * 128);//128M
        CombineTextInputFormat.setInputPaths(job, paths.toArray(new Path[paths.size()]));
        Path tempResultPath = new Path(tempOutPath);
        FileOutputFormat.setOutputPath(job, tempResultPath);

        job.setNumReduceTasks(0);

        boolean flag = job.waitForCompletion(true);

        // 如果成功
        if (flag) {

            // 3.將合并后的文件移動到Hive的分區表
            int index = 0;
            FileStatus[] resultStatus = fs.listStatus(tempResultPath);
            for (FileStatus fileStatus : resultStatus) {
                Path path = fileStatus.getPath();
                if (path.getName().startsWith("part")) {
                    fs.rename(path, new Path(logPath + File.separator + patition + File.separator + currentTimeMillis + "." + index + ".log"));
                    index++;
                }
            }
            fs.delete(tempResultPath, true);

            // 4.標記合并過的文件夾為已經刪除
            for (Path path : paths) {
                fs.rename(path, new Path(path.getParent(), "delete_" + path.getName()));
            }

            fs.close();
        }
    }
}

執行腳本

#!/bin/sh
day=`date '+%Y%m%d'`
echo "提交合并任務 $day"
nohup /opt/soft/hadoop-2.7.3/bin/hadoop jar /opt/soft-install/schedule/mapreduce-1.0.jar me.jinkun.mr.merge.MergeSmallFileAndLoadIntoHive /log/blog day=$day > nohup.log 2>&1 &
?著作權歸作者所有,轉載或內容合作請聯系作者
平臺聲明:文章內容(如有圖片或視頻亦包括在內)由作者上傳并發布,文章內容僅代表作者本人觀點,簡書系信息發布平臺,僅提供信息存儲服務。

推薦閱讀更多精彩內容