List集合中map判断重复数据
场景：导入Excel数据中判断数据中是否有重复数据（注意适用于有唯一列的情况）
//校验文件是否有相同编码的数据
LinkedHashMap tempMap = new LinkedHashMap();
for (int i = 0; i < cdata.size(); i++) {
String key = (String) cdata.get(i).get("equipCode");
String old = (String) tempMap.get(key);
if (old != null){
tempMap.put(key,old + "," + (i + 1));
}else{
tempMap.put(key,"" + (i + 1));
}
}
String err = "";
Iterator it = tempMap.keySet().iterator();
while (it.hasNext()){
String key = (String) it.next();
String value = (String) tempMap.get(key);
if (value.indexOf(",") != -1){
err = err + key + "重复， 检查数据行：" + value + "\n";
}
}
if (StringUtils.isNotBlank(err)){
throw new MyException( "\n" + err);
}
效果
对象集合判断某字段是否有重复
/**
* 判断List<Knowledge>的对象code是否有重复，有重复true
*
* @param orderList
* @return
*/
private Boolean knowledgeIsRepeat(List<Knowledge> orderList) {
Set<Knowledge> set = new TreeSet<Knowledge>(new Comparator<Knowledge>() {
public int compare(Knowledge a, Knowledge b) {
// 字符串则按照asicc码升序排列
return a.getCode().compareTo(b.getCode());
}
});
set.addAll(orderList);
if (set.size() < orderList.size()) {
return true;
}
return false;
}
public static void main(String[] args) {
Person person1 = new Person("小刘", 24, "男");
Person person2 = new Person("小贺", 18, "女");
Person person3 = new Person("小袁", 18, "女");
Person person4 = new Person("小李", 21, "男");
Person person5 = new Person("小子", 21, "男");
List<Person> persons = Arrays.asList(person1, person2, person3, person4,person5);
// 根据指定属性分组，并统计数量（key：指定属性，value：数量）
Map<Object, Long> mapGroup = persons.stream().collect(Collectors.groupingBy(person -> person.getAge(), Collectors.counting()));
System.out.println(mapGroup.toString());
// 筛选Map中value大于1的key
Stream<Object> stringStream = mapGroup.entrySet().stream().filter(entry -> entry.getValue() > 1).map(entry -> entry.getKey());
System.out.print("重复的数据：{ ");
stringStream.forEach(str -> {
System.out.print(str + " ");
});
System.out.println("}");
}
对象集合去除某字段重复数据
public static void main(String[] args) {
List<User> userList = new ArrayList<>();
userList.add(new User("1","李大锤","23","南京"));
userList.add(new User("2","张无忌","18","西安"));
userList.add(new User("3","刘德华","26","苏州"));
userList.add(new User("4","郭靖","33","上海"));
userList.add(new User("1","李大锤","23","南京")); //id相同，其他数据也相同
userList.add(new User("3","带头大哥","36","杭州")); //id相同，其他数据不同
System.out.println(userList);
//根据userid去重
userList = userList.stream().collect(Collectors.collectingAndThen(Collectors.toCollection(() -> new TreeSet<>(Comparator.comparing(User :: getUserid))), ArrayList::new));
System.out.println(userList);
}
List<User> lists = 从某处得来的集合;
lists = lists.stream().collect(Collectors.collectingAndThen(Collectors.toCollection(() -> new TreeSet<>(Comparator.comparing(User::getXxx))), ArrayList::new));
该方法有时返回的集合正确，有时不正确。（慎重使用！）
private static List<User> getDistinctInfo(List<User> list) {
Set<User> set = new TreeSet<>(new Comparator<User>() {
@Override
public int compare(User o1, User o2) {
int compareToResult = 1;//==0表示重复
if(StringUtils.equals(o1.getXxx().toString(), o2.getXxx().toString())) {
compareToResult = 0;
}
return compareToResult;
}
});
set.addAll(list);
return new ArrayList<>(set);
}
最主要的是 distinctByKey 方法，实现了根据集合某个属性去重，在结合Java8的流中的筛选 filter ，最终实现了最简单的List集合根据对象某个属性去重
public class Test {
public static void main(String[] args) {
List<User> list = new Lists.newArrayList();
list.add(new user(1,"张三","上海"));
list.add(new user(2,"李四","北京"));
list.add(new user(3,"王五","济南"));
list.add(new user(4,"李四","北京"));
// 根据list中的City城市来去重
newList = list.stream().filter(distinctByKey(User::getCity)).collect(Collectors.toList());
}
private static <T> Predicate<T> distinctByKey(Function<? super T, Object> keyExtractor) {
Map<Object, Boolean> seen = new ConcurrentHashMap<>();
return t -> seen.putIfAbsent(keyExtractor.apply(t), Boolean.TRUE) == null;
}
}
userList = userList.stream().collect(Collectors.collectingAndThen(Collectors.toCollection(() -> new TreeSet<>(Comparator.comparing(User::getUserid))), ArrayList::new));
public static void main(String[] args) {
List<User> userList = new ArrayList<>();
userList.add(new User("1", "李大锤", "23", "南京"));
userList.add(new User("2", "张无忌", "18", "西安"));
userList.add(new User("3", "刘德华", "26", "苏州"));
userList.add(new User("4", "郭靖", "33", "上海"));
userList.add(new User("1", "李大锤", "23", "南京")); //id相同，其他数据也相同
userList.add(new User("3", "带头大哥", "36", "杭州")); //id相同，其他数据不同
System.out.println("原始数据:" + userList);
//使用stream流,然后可以使用steam中才能用的API
userList = userList.stream().collect(
// collectingAndThen: 第一个参数是 Collectors,第二个参数是Function,用来处理第一个参数
Collectors.collectingAndThen(
// toCollection: 把TreeSet转为Collectors类
Collectors.toCollection(
// 根据userId去重,主要是利用了 TreeSet的有序不重复特性,并且TreeSet能传入比较器,就能根据某一字段去重
() -> new TreeSet<>( Comparator.comparing(
//这里是 User::getUserid 写法的lambda表达式写法
user -> user.getUserid()
))
)
, //这里是 ArrayList::new 的lambda表达式写法,将上面的 TreeSet转为ArrayList
treeSet -> new ArrayList<>(treeSet)
)
);
System.out.println("去重后数据:"+ userList);
}
public static void main(String[] args) {
User u1 = new User("1", "李大锤", "23", "南京");
User u2 = new User("2", "张无忌", "18", "西安");
User u3 = new User("3", "刘德华", "26", "苏州");
User u4 = new User("4", "郭靖", "33", "上海");
User u5 = new User("1", "李大锤", "23", "南京"); //id相同，其他数据也相同
User u6 = new User("3", "带头大哥", "36", "杭州"); //id相同，其他数据不同
ArrayList<User> userList = new ArrayList<>();
userList.add(u1);
userList.add(u2);
userList.add(u3);
userList.add(u4);
userList.add(u5);
userList.add(u6);
System.out.println("原始数据:" + userList);
userList = new ArrayList<>( userList.stream().collect(Collectors.toCollection(() -> new TreeSet<User>(Comparator.comparing(User::getUserid)))) );
System.out.println("去重后数据:"+ userList);
}
/**
* List集合中的对象按照某个字段去重实现
*
*
*/
public class DuplicateOrderTest {
public static void main(String[] args) {
List<Order> orderList = new ArrayList<Order>();
Order order = new Order();
order.setOrderNo("123");
order.setUserId("aa123");
orderList.add(order);
order = new Order();
order.setOrderNo("123");
order.setUserId("bb123");
orderList.add(order);
order = new Order();
order.setOrderNo("33");
order.setUserId("aa123");
orderList.add(order);
List<Order> dataList = removeDuplicateOrder(orderList);
for(Order data : dataList){
System.out.println(data.getOrderNo() + ":" + data.getUserId());
}
}
/**
* 去重
* @param orderList
* @return
*/
private static List<Order> removeDuplicateOrder(List<Order> orderList) {
Set<Order> set = new TreeSet<Order>(new Comparator<Order>() {
public int compare(Order a, Order b) {
// 字符串则按照asicc码升序排列
return a.getUserId().compareTo(b.getUserId());
}
});
set.addAll(orderList);
return new ArrayList<Order>(set);
}
}
执行结果：
123:aa123
33:aa123
java对象集合根据组合条件去重工具
工具
public class ListUtil {
/**
* 对象集合根据条件去重
* @param tList 集合
* @param keyExtractor 指定去重的条件
* @param <T>
* @return
*/
public static <T> List<T> listDeduplication(List<T> tList, Function<? super T, ?> keyExtractor){
if (null != tList) {
return tList.stream().filter(distinctByVariable(u -> keyExtractor.apply(u))).collect(Collectors.toList());
}else {
return null;
}
}
/**
* putIfAbsent() 方法是
* 如果 key对应的value值不存在, key value 添加到 map 中,并返回 null
* 如果 key对应的value值已存在, key value 不再添加到 map 中, 并返回原 value
*
* 故 newKey(这里的newKey对应user对象中的name的值), 如果(newKey, Boolean.TRUE) 在map中已存在,
* putIfAbsent(newKey, Boolean.TRUE) 会返回 Boolean.TRUE (Boolean.TRUE 被final修饰,故其地址值唯一, 可用作比较)
* 然后判断是否等于 null, 返回false, filter接收到结果为false的Predicate并将该值过滤掉
* @param keyExtractor
* @param <T>
* @return
*/
public static <T> Predicate<T> distinctByVariable(Function<? super T, ?> keyExtractor) {
HashMap<Object, Boolean> map = new HashMap<>();
return t->map.putIfAbsent(keyExtractor.apply(t), Boolean.TRUE) == null;
}
}
测试
/**
* 测试, 根据用户id和名字去重
*/
@Test
public void testListDeduplication(){
@Data
@AllArgsConstructor
class User{
private int userId;
private String userName;
}
ArrayList<User> users = new ArrayList<>();
users.add(new User(1,"小王"));
users.add(new User(1,"小王"));
users.add(new User(2,"小王"));
users.add(new User(2,"小王"));
// 根据用户id和用户名去重
List<User> users1 = ListUtil.listDeduplication(users, u -> {
return u.getUserId() + u.getUserName();
});
// 打印检查
users1.forEach(u->{
System.out.println(u.getUserId()+"-"+u.getUserName());
});
}
结果
1-小王
2-小王
字符串集合去重
public static void main(String[] args) {
List strLs = new ArrayList();
strLs.add("1_4");
strLs.add("1_7");
strLs.add("1_7");
strLs.add("1_9");
strLs.add("1_9");
strLs.add("7_7");
strLs.add("9_9");
strLs.add("1_13");
ArrayList<String> newList = new ArrayList<>();
HashMap map= (HashMap) strLs.stream().collect(Collectors.groupingBy(p -> p, Collectors.counting()));
map.keySet().forEach(key -> {
if((Long)map.get(key) > 1){
newList.add((String) key);
}
});
}
JS去重
对象集合通过某字段去重
function unique(arr,field){
var map = {};
var res = [];
for(var i=0; i < arr.length;i++){
if(!map[arr[i][field]]){
map[arr[i][field]]=1;
res.push(arr[i]);
}
}
return res;
}
简单复习前置知识
//获取对象集合某个属性集合（ES6写法）
var data = [
{
a: 1,
b: 2,
c: 3
},
{
a: 4,
b: 5,
c: 6
}
];
data.map(item => item.a)
//集合、数组去重
//https://developer.mozilla.org/zh-CN/docs/Web/JavaScript/Reference/Global_Objects/Array/from
Array.from(new Set(['foo', 'bar', 'baz', 'foo']));
// [ "foo", "bar", "baz" ]
//对象属性值获取方式
obj.field
obj["field"]
使用场景：
三层级联时加载数据如：
var dataList = [
{
line:1, //一级
stationarea:'A',//二级
stationareaName:'A站区',
station:'a'//三级
},
{
line:1,
stationarea:'A',
stationareaName:'A站区',
station:'b'
},
{
line:1,
stationarea:'A',
stationareaName:'A站区',
station:'c',
stationName:'c站'
},
{
line:1,
stationarea:'B',
stationareaName:'B站区',
station:'ba',
stationName:'ba站'
},
{
line:1,
stationarea:'B',
stationareaName:'B站区',
station:'bb',
stationName:'bb站'
},
{
line:2,
stationarea:'2A',
stationareaName:'2A站区',
station:'2a',
stationName:'2a站'
}
]
//一级处理，过滤出集合中的line字段进行字符串拼接即可
let lineList = Array.from(new Set(dataList.map(item => item.elabel))).sort()
lineList.forEach(item => {
let lineName = item + '号线'
let areaList = dataList.filter(obj => {
return obj.elabel == item
})
})
//二级处理： 在一级循环中处理
var map = {};
var stationList = [];
for(var i=0; i < areaList.length;i++){
//将战区集合 的值放到 map对象中做属性，如果属性获取到值说明已经放过了。
//从而创建了一个新的集合
if(!map[areaList[i]["stationarea"]]){
map[areaList[i]["stationarea"]]=1;
stationList.push(areaList[i]);
}
}
js根据数组中对象的某个属性值进行去重
var arr = [
{
from:'张三',
to: '河南'
},
{
from:'王二',
to: '阿里'
},
{
from:'王二',
to: '杭州'
},
{
from:'王二',
to: '山东'
},
]
//有如上数组，想根据数组中的对象的from属性进行去重，如果from一样的话，重复只保留一条。根据ES6属性编写函数代码如下：
function unique(arr1) {
const res = new Map();
return arr1.filter((a) => !res.has(a.from) && res.set(a.from, 1))
}
方法一：
采用对象访问属性的方法，判断属性值是否存在，如果不存在就添加。
方法二：
采用数组中的reduce方法，遍历数组，也是通过对象访问属性的方法
var arr = [{
key: '01',
value: '乐乐'
}, {
key: '02',
value: '博博'
}, {
key: '03',
value: '淘淘'
},{
key: '04',
value: '哈哈'
},{
key: '01',
value: '乐乐'
}];
// 方法1：利用对象访问属性的方法，判断对象中是否存在key
var result = [];
var obj = {};
for(var i =0; i<arr.length; i++){
if(!obj[arr[i].key]){
result.push(arr[i]);
obj[arr[i].key] = true;
}
}
console.log(result); // [{key: "01", value: "乐乐"},{key: "02", value: "博博"},{key: "03", value: "淘淘"},{key: "04", value: "哈哈"}]
// 方法2：利用reduce方法遍历数组,reduce第一个参数是遍历需要执行的函数，第二个参数是item的初始值
var obj = {};
arr = arr.reduce(function(item, next) {
obj[next.key] ? '' : obj[next.key] = true && item.push(next);
return item;
}, []);
console.log(arr); // [{key: "01", value: "乐乐"},{key: "02", value: "博博"},{key: "03", value: "淘淘"},{key: "04", value: "哈哈"}]
https://www.cnblogs.com/aoeiuv/p/5911692.html
https://blog.csdn.net/yinni11/article/details/88663713
https://blog.csdn.net/weixin_42561607/article/details/106711225
https://www.zhihu.com/question/275792654/answer/635430792
https://segmentfault.com/a/1190000009045039
https://www.jianshu.com/p/131ca13e7f28
https://www.cnblogs.com/xingguozhiming/p/10296727.html
https://blog.csdn.net/yjltx1234csdn/article/details/93766707
https://segmentfault.com/q/1010000019157875?utm_source=tag-newest
https://juejin.cn/post/6844903966145413128
https://www.cnblogs.com/pzw23/p/13570011.html
https://bbs.csdn.net/topics/390262292
174
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
