计算json文件的电影总分
package com.doit.sumrate;
public class UserRateBean {
private String movie;
private Integer rate;
private String timeStamp;
private String uid;
public String getMovie() {
return movie;
}
public void setMovie(String movie) {
this.movie = movie;
}
public Integer getRate() {
return rate;
}
public void setRate(Integer rate) {
this.rate = rate;
}
public String getTimeStamp() {
return timeStamp;
}
public void setTimeStamp(String timeStamp) {
this.timeStamp = timeStamp;
}
public String getUid() {
return uid;
}
public void setUid(String uid) {
this.uid = uid;
}
@Override
public String toString() {
return "UserRateBean{" +
"movie='" + movie + '\'' +
", rate=" + rate +
", timeStamp='" + timeStamp + '\'' +
", uid='" + uid + '\'' +
'}';
}
}
package com.doit.sumrate;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.textoutputFormat;
import org.codehaus.jackson.map.ObjectMapper;
import java.io.IOException;
public class UserRateDriver {
public static class UserRateMap extends Mapper<LongWritable,Text, Text, IntWritable>{
ObjectMapper objectMapper = new ObjectMapper();
protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
String line = value.toString();
//{"movie":"1193","rate":"5","timeStamp":"978300760","uid":"1"}
UserRateBean userRateBean = objectMapper.readValue(line, UserRateBean.class);
String movie = userRateBean.getMovie();
Integer rate = userRateBean.getRate();
context.write(new Text(movie),new IntWritable(rate));
}
}
public static class UserRateReducer extends Reducer<Text, IntWritable,Text, IntWritable> {
protected void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException {
int sum = 0;
for (IntWritable v : values) {
sum = sum + v.get();
}
context.write(key, new IntWritable(sum));
}
}
public static void main(String[] args) throws InterruptedException, IOException, ClassNotFoundException {
Configuration conf = new Configuration();
conf.set("yarn.resorcemanager.hostname", "node01");
conf.set("fs.deafutFS", "hdfs://node01:9000/");
Job job = Job.getInstance(conf);
job.setJarByClass(UserRateDriver.class);
job.setMapperClass(UserRateMap.class);
job.setReducerClass(UserRateReducer.class);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(IntWritable.class);
job.setoutputKeyClass(Text.class);
job.setoutputValueClass(IntWritable.class);
job.setInputFormatClass(TextInputFormat.class);
job.setoutputFormatClass(textoutputFormat.class);
FileInputFormat.setInputPaths(job,new Path(args[0]));
FileOutputFormat.setoutputPath(job,new Path(args[1]));
job.submit();
boolean b = job.waitForCompletion(true);
System.exit(b?0:1);
}
}
[root@node01 home]# hadoop fs -mkdir -p /lll/yyy/input/
[root@node01 home]# hadoop fs -put rating.json /lll/yyy/input/
[root@node01 home]# hadoop fs -cat /lll/yyy/input/rating.json
[root@node01 home]# hadoop jar movitrate.jar /lll/yyy/input/rating.json /lll/yyy/output/
[root@node01 home]# hadoop fs -ls /lll/yyy/input/rating.json/
[root@node01 home]# hadoop fs -cat /lll/yyy/output/part-r-00000/
469 204
47 4669
470 66
471 2175
472 119
473 336
474 3718
475 1217
476 94
477 676
478 59
479 372
48 1137
480 10057
481 994
482 621
483 277
484 77
485 1254
486 180
487 162
488 132
489 240
49 101
490 615
491 703
492 1246
493 1075
494 1392
495 142
496 119
497 2668
498 129
499 135
5 890
50 8054
500 2866
501 339
502 300
503 48
504 526
505 172
506 398
507 1085
508 2185
509 2234
510 101
511 472
512 514
513 209
514 1222
515 1763
516 712
517 1213
518 366
519 392
52 1569
520 1096
521 394
522 296
523 428
524 1721
525 67
526 20
527 10392
528 171
529 2864
53 38
530 7
531 1274
532 1186
533 555
534 1097
535 1149
536 174
537 637
538 1204
539 3708
54 105
540 375
541 7692
542 551
543 2308
544 341
545 2
546 656
547 357
548 349
549 600
55 138
550 537
551 3703
552 1714
553 2142
554 68
555 2585
556 558
557 9
558 365
559 15
56 18
560 29
561 47
562 1816
563 55
564 102
565 191
566 38
567 50
568 135
569 293
57 337
570 74
571 122
572 6
573 74
574 603
575 302
576 3
577 163
578 9
579 2
58 2051
580 201
581 571
582 6
583 103
584 4
585 1215
586 2116
587 3930
588 5118
589 10751
59 22
590 5681
591 4
592 5153
593 11219
594 2934
595 4119
596 1868
597 3809
598 34
599 1079
6 3646
60 1147
600 90
601 3
602 169
603 174
605 749
606 207
607 4
608 10692
609 186
61 201
610 2192
611 534
612 190
613 298
614 24
615 271
616 932
617 62
618 19
619 83
62 2000
621 156
623 9
624 4
626 43
627 436
628 1890
63 317
630 20
631 156
632 52
633 72
634 5
635 279
637 356
638 61
639 204
64 179
640 190
641 1
642 2
643 6
644 2
645 72
647 1567
648 5242
649 67
65 295
650 309
651 2
652 26
653 1974
655 1
656 100
657 6
658 3
659 93
66 162
660 3
661 1819
662 382
663 902
664 24
665 128
666 10
667 96
668 207
669 96
67 13
670 247
671 1678
672 4
673 1475
674 1164
678 1299
679 7
68 207
680 215
681 59
682 14
684 1
685 99
687 9
688 485
69 1166
690 4
691 375
692 224
694 373
695 429
696 13
697 217
698 11
7 1562
70 2885
700 482
701 4
702 99
703 16
704 97
705 138
706 5
707 849
708 2332
709 438
71 202
710 119
711 155
712 7
714 595
715 188
716 75
717 4
718 115
719 684
72 338
720 1939
722 103
724 1284
725 254
726 27
728 817
729 11
73 855
730 1
731 134
732 17
733 4989
734 9
735 339
736 3523
737 609
74 357
741 1284
742 440
743 275
744 6
745 2970
746 124
747 110
748 1314
749 11
75 23
750 6083
751 53
753 101
754 38
755 34
756 12
757 15
758 4
759 242
76 508
760 116
761 343
762 651
763 3
764 242
765 450
766 357
767 77
769 39
77 128
771 68
774 4
775 29
776 12
778 2974
779 140
78 163
780 6073
781 348
782 370
783 1257
784 1051
785 2661
786 1437
787 15
788 2840
789 6
79 297
790 3
791 7
792 4
793 26
796 10
798 552
799 1132
8 205
80 199
800 2572
801 375
802 1493
803 210
804 694
805 1235
806 190
807 42
808 146
809 270
81 525
810 176
811 35
813 118
814 3
815 12
818 484
82 345
820 13
821 4
823 13
824 110
826 1
827 6
828 157
829 300
83 122
830 871
831 77
832 1962
833 170
834 8
835 322
836 629
837 758
838 1944
839 299
84 65
840 128
841 85
842 391
843 1
844 25
846 142
847 13
848 536
849 1283
85 660
850 110
851 444
852 1981
853 12
854 29
858 10059
859 2
86 801
860 36
861 959
862 113
863 35
864 43
865 2
866 2187
867 53
868 3
869 130
87 168
870 107
872 3
874 25
875 149
876 60
877 79
878 4
879 670
88 565
880 713
881 202
882 160
884 6
885 111
886 306
887 5
888 106
889 5
89 704
891 432
892 587
893 255
895 2
896 175
897 368
898 2503
899 3217
9 271
90 22
900 1321
901 634
902 2642
903 3867
904 4700
905 1601
906 766
907 412
908 5765
909 1721
910 3578
911 1280
912 7365
913 4585
914 2642
915 1447
916 1792
917 283
918 717
919 7298
92 206
920 4621
921 1176
922 2111
923 4898
924 6982
925 39
926 1715
927 334
928 1623
929 352
93 199
930 1911
931 806
932 747
933 1817
934 1112
935 258
936 538
937 165
938 677
939 35
94 678
940 1502
941 389
942 1227
943 681
944 368
945 1041
946 620
947 772
948 488
949 855
95 1835
950 1238
951 1687
952 969
953 3134
954 1624
955 1722
956 142
957 120
958 44
959 217
96 51
960 69
961 80
962 39
963 193
964 173
965 1031
966 24
967 75
968 2625
969 4494
97 165
970 126
971 1097
972 19
973 321
974 60
975 3
976 200
977 4
978 394
98 12
980 14
981 11
982 298
984 88
985 15
986 836
987 116
988 233
989 5
99 173
990 139
991 599
992 52
993 28
994 1843
996 744
997 94
998 280
999 939
........
........
........
