[Hadoop@dtydb6 logs]$ vi hadoop-hadoop-datanode-dtydb6.log
at java.io.RandomAccessFile.<init>(RandomAccessFile.java:233)
at org.apache.hadoop.hdfs.server.datanode.FSDataset.getBlockInputStream(FSDataset.java:1094)
at org.apache.hadoop.hdfs.server.datanode.BlockSender.<init>(BlockSender.java:168)
at org.apache.hadoop.hdfs.server.datanode.BlockSender.<init>(BlockSender.java:81)
at org.apache.hadoop.hdfs.server.datanode.DataBlockScanner.verifyBlock(DataBlockScanner.java:453)
at org.apache.hadoop.hdfs.server.datanode.DataBlockScanner.verifyFirstBlock(DataBlockScanner.java:519)
at org.apache.hadoop.hdfs.server.datanode.DataBlockScanner.run(DataBlockScanner.java:617)
at java.lang.Thread.run(Thread.java:722)
2013-02-17 00:00:29,023 WARN org.apache.hadoop.hdfs.server.datanode.DataBlockScanner: Second Verification failed for blk_1408462853104263034_39617. Exception : java.io.FileNotFoundException: /hadoop/logdata/current/subdir2/subdir2/blk_1408462853104263034 (Too many open files)
at java.io.RandomAccessFile.open(Native Method)
at java.io.RandomAccessFile.<init>(RandomAccessFile.java:233)
at org.apache.hadoop.hdfs.server.datanode.FSDataset.getBlockInputStream(FSDataset.java:1094)
at org.apache.hadoop.hdfs.server.datanode.BlockSender.<init>(BlockSender.java:168)
at org.apache.hadoop.hdfs.server.datanode.BlockSender.<init>(BlockSender.java:81)
at org.apache.hadoop.hdfs.server.datanode.DataBlockScanner.verifyBlock(DataBlockScanner.java:453)
at org.apache.hadoop.hdfs.server.datanode.DataBlockScanner.verifyFirstBlock(DataBlockScanner.java:519)
at org.apache.hadoop.hdfs.server.datanode.DataBlockScanner.run(DataBlockScanner.java:617)
at java.lang.Thread.run(Thread.java:722)
2013-02-17 00:00:29,023 INFO org.apache.hadoop.hdfs.server.datanode.DataBlockScanner: Reporting bad block blk_1408462853104263034_39617 to namenode.
2013-02-17 00:00:53,076 WARN org.apache.hadoop.hdfs.server.datanode.DataBlockScanner: First Verification failed for blk_4328439663130931718_44579. Exception : java.io.FileNotFoundException: /hadoop/logdata/current/subdir9/subdir12/blk_4328439663130931718 (Too many open files)
at java.io.RandomAccessFile.open(Native Method)
at java.io.RandomAccessFile.<init>(RandomAccessFile.java:233)
at org.apache.hadoop.hdfs.server.datanode.FSDataset.getBlockInputStream(FSDataset.java:1094)
at org.apache.hadoop.hdfs.server.datanode.BlockSender.<init>(BlockSender.java:168)
at org.apache.hadoop.hdfs.server.datanode.BlockSender.<init>(BlockSender.java:81)
at org.apache.hadoop.hdfs.server.datanode.DataBlockScanner.verifyBlock(DataBlockScanner.java:453)
at org.apache.hadoop.hdfs.server.datanode.DataBlockScanner.verifyFirstBlock(DataBlockScanner.java:519)
at org.apache.hadoop.hdfs.server.datanode.DataBlockScanner.run(DataBlockScanner.java:617)
at java.lang.Thread.run(Thread.java:722)
2013-02-17 00:00:53,077 WARN org.apache.hadoop.hdfs.server.datanode.DataBlockScanner: Second Verification failed for blk_4328439663130931718_44579. Exception : java.io.FileNotFoundException: /hadoop/logdata/current/subdir9/subdir12/blk_4328439663130931718 (Too many open files)
at java.io.RandomAccessFile.open(Native Method)
at java.io.RandomAccessFile.<init>(RandomAccessFile.java:233)
at org.apache.hadoop.hdfs.server.datanode.FSDataset.getBlockInputStream(FSDataset.java:1094)
at org.apache.hadoop.hdfs.server.datanode.BlockSender.<init>(BlockSender.java:168)
at org.apache.hadoop.hdfs.server.datanode.BlockSender.<init>(BlockSender.java:81)
at org.apache.hadoop.hdfs.server.datanode.DataBlockScanner.verifyBlock(DataBlockScanner.java:453)
at org.apache.hadoop.hdfs.server.datanode.DataBlockScanner.verifyFirstBlock(DataBlockScanner.java:519)
at org.apache.hadoop.hdfs.server.datanode.DataBlockScanner.run(DataBlockScanner.java:617)
at java.lang.Thread.run(Thread.java:722)
2013-02-17 00:00:53,077 INFO org.apache.hadoop.hdfs.server.datanode.DataBlockScanner: Reporting bad block blk_4328439663130931718_44579 to namenode.
2013-02-17 00:01:10,115 WARN org.apache.hadoop.hdfs.server.datanode.DataBlockScanner: First Verification failed for blk_2833765807455012512_10228. Exception : java.io.FileNotFoundException: /hadoop/logdata/current/subdir63/subdir25/blk_2833765807455012512 (Too many open files)
at java.io.RandomAccessFile.open(Native Method)
at java.io.RandomAccessFile.<init>(RandomAccessFile.java:233)
at org.apache.hadoop.hdfs.server.datanode.FSDataset.getBlockInputStream(FSDataset.java:1094)
at org.apache.hadoop.hdfs.server.datanode.BlockSender.<init>(BlockSender.java:168)
at org.apache.hadoop.hdfs.server.datanode.BlockSender.<init>(BlockSender.java:81)
at org.apache.hadoop.hdfs.server.datanode.DataBlockScanner.verifyBlock(DataBlockScanner.java:453)
at org.apache.hadoop.hdfs.server.datanode.DataBlockScanner.verifyFirstBlock(DataBlockScanner.java:519)
网络搜索，怀疑linux nofile超过最大限制，当前设置大小1024，默认值